AWS Lambda Get Image and Upload to S3

Question

I am working in a AWS Lambda function. I am successfully making an API call to the NASA APOD and getting back the values. I want to take the url for the image and download that image and then upload into S3. I am getting an error when I try to access the "test.jpg" image, "Error: EACCES: permission denied, open 'test.jpg'". If I move the S3bucket.putObject outside the http.request, I get data is equal to null. I know I am missing something simple. Thought?

function GetAPOD(intent, session, callback) {
    var nasa_api_key = 'demo-key'
    ,   nasa_api_path = '/planetary/apod?api_key=' + nasa_api_key;

    var options = {
        host: 'api.nasa.gov',
        port: 443,
        path: nasa_api_path,
        method: 'GET'
    };

    var req = https.request(options, function (res) {
        res.setEncoding('utf-8');

        var responseString = '';
        res.on('data', function (data) {
            responseString += data;
        });

        res.on('end', function () {
            console.log('API Response: ' + responseString);

            var responseObject = JSON.parse(responseString)
            ,   image_date = responseObject['date']
            ,   image_title = responseObject['title']
            ,   image_url = responseObject['url']
            ,   image_hdurl = responseObject['hdurl']
            ,   image_desc = responseObject['explanation'];

            var s3Bucket = new AWS.S3( { params: {Bucket: 'nasa-apod'} } );

            var fs = require('fs');

            var file = fs.createWriteStream("test.jpg");
            var request = http.get(image_url, function(response) {
                response.pipe(file);

                var data = {Key: "test.jpg", Body: file};
                s3Bucket.putObject(data, function(err, data) {
                    if (err) {
                        console.log('Error uploading data: ', data); 
                    }
                    else {
                        console.log('succesfully uploaded the image!');
                    }
                });
            });

        });
    });

    req.on('error', function (e) {
        console.error('HTTP error: ' + e.message);
    });

    //req.write();
    req.end();
}

How do you try to access that image or what method your are using to access that image? — Piyush Patil
– Piyush Patil, Commented Jun 7, 2016 at 15:34
I am accessing the file variable in the s3Bucket.putObject as the data object's Body. — Kris
– Kris, Commented Jun 7, 2016 at 15:42

Mark B · Accepted Answer · 2016-06-07 15:48:09Z

1

You need to be writing the file to /tmp. That's the only directory in the Lambda environment that you will have write access to.

answered Jun 7, 2016 at 15:48

Mark B

203k27 gold badges339 silver badges332 bronze badges

Sign up to request clarification or add additional context in comments.

4 Comments

Kris Over a year ago

Mark - Thanks, I was able to write the file and not get any errors regarding the test.jpg. When the putObject is called it returns "NetworkingError: Cannot pipe. Not readable." One step closer!

Mark B Over a year ago

You are passing a write stream to putObject. I think you need to close the stream, and create a new read stream. Alternatively, I don't think you really need to write the data to a temp file at all since you have it in memory already.

Kris Over a year ago

Thanks, that makes sense. Would the request variable be the data in memory or the response? I tried both and get different error messages.

Kris Over a year ago

I solved it and posted an answer, thank you for the help.

Kris · Accepted Answer · 2016-06-07 18:14:04Z

I got it!! Thank you Mark B for the help. I was able to get the data from the stream without saving it locally and then writing to the bucket. I did have to change my IAM role to allow the putObject for S3.

function GetAPOD(intent, session, callback) {
    var nasa_api_key = 'demo-key'
    ,   nasa_api_path = '/planetary/apod?api_key=' + nasa_api_key;

    var options = {
        host: 'api.nasa.gov',
        port: 443,
        path: nasa_api_path,
        method: 'GET'
    };

    var req = https.request(options, function (res) {
        res.setEncoding('utf-8');

        var responseString = '';
        res.on('data', function (data) {
            responseString += data;
        });

        res.on('end', function () {
            // console.log('API Response: ' + responseString);

            var responseObject = JSON.parse(responseString)
            ,   image_date = responseObject['date']
            ,   image_title = responseObject['title']
            ,   image_url = responseObject['url']
            ,   image_hdurl = responseObject['hdurl']
            ,   image_desc = responseObject['explanation'];

            var image_name = image_date + '.jpg';

            var s3 = new AWS.S3();
            var s3Bucket = new AWS.S3( { params: {Bucket: 'nasa-apod'} } );

            var request = http.get(image_url, function(response) {
                var image_stream = null;
                response.on('data', function (data) {
                    image_stream = data;
                });

                response.on('end', function () {
                    var param_data = {Key: image_name, Body: image_stream, ContentType: "image/jpeg", ContentLength: response.headers['content-length']};
                    s3Bucket.putObject(param_data, function(err, output_data) {
                        if (err) {
                            console.log('Error uploading data to S3: ' + err); 
                        }
                    });
                });
            });
            request.end();

        });
    });

    req.on('error', function (e) {
        console.error('HTTP error: ' + e.message);
    });

    req.end();
}

I thought I had it, it puts a file in my S3 bucket but the file is not valid when I try and open it.
I had to add the ContentType and ContentLength to the param_data object. It is now working.

Collectives™ on Stack Overflow

AWS Lambda Get Image and Upload to S3

2 Answers 2

4 Comments

2 Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

4 Comments

2 Comments

Your Answer

Sign up or log in

Post as a guest

Related