Multiple file upload to s3 node.js

Question

I am working on MEAN stack application and I am using AWS SDK to upload multiple files to S3. I am using busboy and AWS SDK.

Code:

var inputObj = {};
 var busboy = new Busboy({ headers: req.headers });
    // The file upload has completed
    busboy.on('finish', function() {
     console.log('Upload finished.....');
     var file = [];
     const file1 = req.files.clogo;
     const file2 = req.files.cbanner1;
     const file3 = req.files.cbanner2;
     const file4 = req.files.cbanner3;
     const file5 = req.files.cbanner4;
     const file6 = req.files.clongHeader;
     file.push(file1);
     file.push(file2);
     file.push(file3);
     file.push(file4);
     file.push(file5);
     file.push(file6);
     multipleUploadToS3(req.body.cname, file, function(fileName) {
            console.log("client file upload finished.....");
             if(fileName.length == 6){
                inputObj.clogo = fileName[0];
                inputObj.cbanner1 = fileName[1];
                inputObj.cbanner2 = fileName[2];
                inputObj.cbanner3 = fileName[3];
                inputObj.cbanner4 = fileName[4];
                inputObj.clongHeader = fileName[5];

                console.log(inputObj);
                var clientObj = new client(inputObj);
                clientObj.save(function(err, client) {
                    console.log("Client Saved.....");
                    if (err) {
                        return res.status(400).send({
                            message: errorHandler.getErrorMessage(err)
                        });
                    } else {
                        res.json(client);
                    }
                });    
             }
     });
    });
    req.pipe(busboy);

File Upload Method:

function multipleUploadToS3(client, file, callback) {
    console.log("multipleUpload to S3"); 
    console.log(client);
    console.log(file);

    let s3bucket = new AWS.S3({
        accessKeyId: IAM_USER_KEY,
        secretAccessKey: IAM_USER_SECRET,
        Bucket: BUCKET_NAME,
    });

    var fileNames = [];

    for(var i=0; i<file.length; i++){
        s3bucket.createBucket(function () {
            var params = {
            Bucket: BUCKET_NAME,
            Key: client+ '/' + file[i].name,
            Body: file[i].data,
            };
            s3bucket.upload(params, function (err, data) {
                if (err) {
                    console.log('error in callback');
                    console.log(err);
                }
                console.log('success');
                //console.log(data.key);
                fileNames.push(data.key);
                if(i == file.length){ callback(fileNames);}
            });
        });
    }

};

The issue: file upload is asynchronous so for example if file1 I am uploading is honest.jpg then I want multipleUploadToS3 method to return file name after its done uploading to S3. I am binding this in inputObj keys which will be saved to mongo db. so inputObj.logo should have logo.png in it not the banner image which is happening due to asynchronous call.

This is working for a single file but failing for multiple files.

Saurabh Ghewari · Accepted Answer · 2018-08-08 12:46:36Z

2

The problem is because for loop is synchronous and file upload is asynchronous.

Take a look at this example below,

for(var i = 0; i<5;i++) {
  setTimeout(function(){ console.log(i); }, 100);
}

The above loop will print 5 for 5 times i.e 55555.

This behaviour is because for loop gets executed immediately making i=5 and when timeout gets executed it prints "i" value 5 for 5 times. 5 times because setTimeout is pushed in the queue for five times.

There are two ways to solve the problem you are facing

You can use Recursion.
Use neo-async(async-parallel) lib to control the async flow of javascript(Nodejs). click here for neo-async lib

Hope this clears your doubt. Please comment for more info.

edited Aug 8, 2018 at 12:46

answered Aug 8, 2018 at 12:39

Saurabh Ghewari

7093 silver badges8 bronze badges

Sign up to request clarification or add additional context in comments.

Collectives™ on Stack Overflow

Multiple file upload to s3 node.js

1 Answer 1

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

Comments

Your Answer

Sign up or log in

Post as a guest

Related