Append string to a text file Nodejs in AWS Lambda

Question

The scenario: A text file snapshot-ids.txt is located in a S3 bucket. I'm trying to create a Lambda function that run daily (Cron) that would use AWS CLI to take snapshot of a volume, then save that snapshotId to a text file in S3. On the next time another snapshot is created, the new snapshotId will be saved to the same text file on S3. The text file is a place holder for snapshotIds and when it reaches a threshold, it will delete the top snapshotIds and add the new one at the end (FIFO pipe).

For people who don't use AWS lambda, my question is what's the quickest way to append text to a variable and return the new variable with new lines in it.

For people who know Lambda, this is the basic code from AWS Lambda I have, I use fs.appendFile, but how do I use the file I got from s3.getObject() and eventually pass it to s3.putObject()?

EDIT: this is my progress:

console.log('Loading function');

var aws = require('aws-sdk');
var s3 = new aws.S3({ apiVersion: '2006-03-01' });
var fs = require('fs');

exports.handler = function(event, context) {
    //console.log('Received event:', JSON.stringify(event, null, 2));

    // Get the object from the event and show its content type
    var bucket = event.Records[0].s3.bucket.name;
    var key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
    var params = {
        Bucket: bucket,
        Key: key
    };
    s3.getObject(params, function(err, data) {
        if (err) {
            console.log(err);
            var message = "Error getting object " + key + " from bucket " + bucket +
                ". Make sure they exist and your bucket is in the same region as this function.";
            console.log(message);
            context.fail(message);
        } else {
            // fs.appendFile('snapshot-ids.txt', 'snap-001', function (err) {
            //     if (err) throw err;
            //     console.log('The "data to append" was appended to file!');
            // });
            console.log(params_new);
            console.log('CONTENT TYPE getObject:', data.ContentType);
            // context.succeed(data.Body.toString('ascii'));
        }
    });
    var params_new = {
        Bucket: bucket,
        Key: key,
        Body: 'snap-002'
    };
    s3.putObject(params_new, function(err, data) {
                console.log('put here');
                if (err) {
                    console.log(err);
                    var message = "Error getting object " + key + " from bucket " + bucket +
                        ". Make sure they exist and your bucket is in the same region as this function.";
                    console.log(message);
                    context.fail(message);
                } else {
                    console.log('CONTENT TYPE putObject:', data.ContentType);
                    context.succeed(data.ContentType);
                }
    });
};

Ryan · Accepted Answer · 2015-12-03 16:42:20Z

2

A couple of things I noticed with your code so far...

You can't call s3.putObject until s3.getObject is finished and you have the file from s3.
You aren't dealing with the file system since you get the data from s3.getObject.

With those things in mind I modified your code(I haven't tried this but it should get you going in the right direction):

console.log('Loading function');

var aws = require('aws-sdk');
var s3 = new aws.S3({ apiVersion: '2006-03-01' });

exports.handler = function(event, context) {
    //console.log('Received event:', JSON.stringify(event, null, 2));

    // Get the object from the event and show its content type
    var bucket = event.Records[0].s3.bucket.name;
    var key = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' '));
    var params = {
        Bucket: bucket,
        Key: key
    };
    s3.getObject(params, function(err, data) {
        if (err) {
            console.log(err);
            var message = "Error getting object " + key + " from bucket " + bucket +
                ". Make sure they exist and your bucket is in the same region as this function.";
            console.log(message);
            context.fail(message);
        } else {
            console.log(params_new);
            console.log('CONTENT TYPE getObject:', data.ContentType);

            // convert body(file contents) to a string so we can append
            var body = data.Body.toString('utf-8');
            // append data
            body += 'snap-001\n';

            var params_new = {
                Bucket: bucket,
                Key: key,
                Body: body
            };
            //NOTE this call is now nested in the s3.getObject call so it doesn't happen until the response comes back
            s3.putObject(params_new, function(err, data) {
                        console.log('put here');
                        if (err) {
                            console.log(err);
                            var message = "Error getting object " + key + " from bucket " + bucket +
                                ". Make sure they exist and your bucket is in the same region as this function.";
                            console.log(message);
                            context.fail(message);
                        } else {
                            console.log('CONTENT TYPE putObject:', data.ContentType);
                            context.succeed(data.ContentType);
                        }
            });

        }
    });

};

Something else to keep in mind is if you have more than 1 of this Lambda running at the same time it is likely they will stomp on each others changes. Sounds like you will just be scheduling it once a day so it shouldn't be a big deal but its worth noting.

answered Dec 3, 2015 at 16:42

Ryan

5,99334 silver badges27 bronze badges

Sign up to request clarification or add additional context in comments.

7 Comments

Casper Over a year ago

Thanks for the answer, this works perfect, btw do you know if Nodejs in Lambda supports AWS CLI?

Ryan Over a year ago

I'm not sure about accessing the CLI in Lambda but the SDK has most of the CLI functionality: docs.aws.amazon.com/AWSJavaScriptSDK/latest/index.html

Casper Over a year ago

Thanks, this got me starting well. But I'm now stuck at how to access EC2 from Lambda, even though I created a role for temporary access but it seems impossible

Casper Over a year ago

When I used ec2.creatsnapshot() before calling s3.putObject(), the function worked (created a snapshot) but I don't have the return value (data.snapshotId)

Ryan Over a year ago

Do you get anything back when you console.log data? According to the docs it looks like it should be data.SnapshotId. Notice the capital S.

|

Collectives™ on Stack Overflow

Append string to a text file Nodejs in AWS Lambda

1 Answer 1

7 Comments

Your Answer

Linked

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

7 Comments

Your Answer

Sign up or log in

Post as a guest

Linked

Related