Read JSON from S3 file and Insert records into dynamoDB using Lambda with NodeJS runtime

Question

My DynamoDB table has the following partition key:

UserId  String

This is my Lambda function:

'use strict';

console.log('Loading function');
const doc = require('dynamodb-doc');
const dynamo = new doc.DynamoDB();

var AWS = require('aws-sdk');

var S3 = new AWS.S3({
    maxRetries: 0,
    region: 'us-east-1',
});

var insertSuccess = 0;
var insertErrors = 0;

function dynamoResultCallback(err, data) {
    if (err) {
        insertErrors++;
        console.log("Insert Error: \n");
        console.log(err, err.stack); // an error occurred
    } else {
        insertSuccess++;
    }
}

exports.handler = (event, context, callback) => {
    console.log('Received event:', JSON.stringify(event, null, 2));
    console.log("Init complete, running.. \n")

    var srcBucket = event.Records[0].s3.bucket.name;
    var srcKey = event.Records[0].s3.object.key;

    console.log("Params: srcBucket: " + srcBucket + " srcKey: " + srcKey + "\n")

    S3.getObject({
        Bucket: srcBucket,
        Key: srcKey,
    }, function(err, data) {
        if (err !== null) {
            return callback(err, null);
        }
        var fileData = data.Body.toString('utf-8');
        var recordsArray = fileData.split("\n");

        for (var i = 0; i < recordsArray.length; i++) {
             var record = recordsArray[i];
              console.log("Inserting record: " + record);

            var params = {
                Item: record,
                ReturnConsumedCapacity: "TOTAL",
                TableName: "PacketData"
            };
            dynamo.putItem(params, dynamoResultCallback);
        }
        console.log("Insert Result -- successCount: " + insertSuccess + " errorCount: " + insertErrors)
        return callback(null, data);
    });
};

What's happening currently is that, it reads the S3 file as expected, but I'm not able to pass the resulting records into dynamo, the cloud watchlog error is that it didn't get passed a UserId (the required parition key) despite it being in the data. The records print fine in CloudWatch, and actually if I replace the record variable on this line:

Item: record,

With the JSON string from CloudWatch it works as expected. Any suggestions?

P.S. I'm a JavaScript novice if that wasn't already apparent, but I believe the problem is due to how the variable record is being interpreted at runtime as it works fine if I replace that variable with it's value as a hard coded string.

Can you try JSON.parse(record); converting the record string to an object? — Ashan
– Ashan, Commented Jul 2, 2017 at 23:09
Ah, yes that was the problem, JSON.parse(record) fixed it. Thank you! — RandomUser
– RandomUser, Commented Jul 3, 2017 at 2:41

Ashan · Accepted Answer · 2017-07-03 03:19:02Z

5

Use JSON.parse(record);

According to the dynamodb-doc documentation it putItem method expects an object rather a string.

answered Jul 3, 2017 at 3:19

Ashan

19.7k5 gold badges53 silver badges69 bronze badges

Sign up to request clarification or add additional context in comments.

Collectives™ on Stack Overflow

Read JSON from S3 file and Insert records into dynamoDB using Lambda with NodeJS runtime

1 Answer 1

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

Comments

Your Answer

Sign up or log in

Post as a guest

Related