running multiple async functions sequentially in node.js

Question

I'm new to javascript also node.js environment. It's hard for me to use the call back. Because when I have a lot of procedure linked together, it's hard to write and it will becomes very nasty.

So, after I google for the answer, I found the async tool, but it still run like that it can execute the codes block by block using call back. for example, f1 -> f2 -> f3 ...

however, we always has a delay for the server and my computer. then If I do some thing like create buckets, then print how many I got. It will print out the original amount of buckets not after I add one. Same situation like delete bucket .... can some god help me

thanks...

function bucketSerialTesting(s3)
{
    async.series([
      function f1(callback)
      {
          settingBucket ('testjsbucket2',s3);
          callback('1');
      }
      ,
      function f2(callback)
      {
        settingBucket('testjsbucket3', s3);
        callback('2');
      }
      ,
      function f3(callback)
      {
        listBucket(s3);
        callback('3');
      }
      ,
      function f4(callback)
      {
          deleteBucket('testjsbucket2', s3);
          callback('4');
      }
      ,
      function f5(callback)
      {
          deleteBucket('testjsbucket3', s3);
          callback('5');
      }
      ,
      function f6(callback)
      {
          listBucket(s3);
          callback('6');
      },
    ],
    function(err, results){
        console.log(results);
    });
}



function settingBucket (bucketName, s3){
    var s3 = new AWS.S3();
    var params = {
          Bucket: bucketName, /* required */
    };
    s3.createBucket(params, function(err, data) {
     if (err) console.log(err, err.stack); // an error occurred
     else     console.log(data);           // successful response
    }).on('success',function(response)
            //put bucket cors for browsers javascript testing
            {
                var params = {
                Bucket: bucketName, /* required */
                CORSConfiguration: {
                CORSRules: [
                  {
                    AllowedHeaders: ['*',],
                    AllowedMethods: ['GET','PUT','DELETE','POST'],
                    AllowedOrigins: ['*','null'],
                    ExposeHeaders:  ['ETag',],
                   },
                ]
                },
                };
                s3.putBucketCors(params, function(err, data) {
                if (err) console.log(err, err.stack); // an error occurred
                else     {
                    console.log(data);
                    console.log('put bucketcors succeed');
                }
                });
            }).send();
}




function listBucket(s3)
{
    s3.listBuckets(function(err, data) {
          if (err) console.log(err, err.stack); // an error occurred
          else     {
            console.log(data);
            console.log('we have '+data.Buckets.length.toString()+' buckets');
            for (var i = data.Buckets.length - 1; i >= 0; i--) {
                console.log(data.Buckets[i].Name);
            };
          }           // successful response
        });
}

function deleteBucket(bucketName, s3){
    var params = {
          Bucket: bucketName, /* required */
        };
        s3.listObjectVersions(params, function(err, data) {
          if (err) console.log(err, err.stack); // an error occurred
          else{
              console.log(data);           
              params = {Bucket: bucketName};
                params.Delete = {};
                params.Delete.Objects = [];
                console.log(data.Versions.length);
                for(j = 0; j < data.Versions.length;++j)
                {
                    params.Delete.Objects.push({Key: data.Versions[j].Key,
                    VersionId: data.Versions[j].VersionId
                    });
                }
                for(j = 0; j < data.DeleteMarkers.length;++j)
                {
                    params.Delete.Objects.push({Key: data.DeleteMarkers[j].Key,
                    VersionId: data.DeleteMarkers[j].VersionId
                    });
                }
                s3.deleteObjects(params, function(err, data) {
                  if (err) console.log(err, err.stack); // an error occurred
                  else     {
                      console.log(data);
                  }
                  params = {Bucket: bucketName};
                  s3.deleteBucket(params, function(err, data) {
                      if (err) console.log(err, err.stack); // an error occurred
                      else     console.log(data);           // successful response
                    });     
                });

          }
        });
}

Neither of those functions you are calling seems to use callbacks. Why do you think you need async.js? Or are you just calling them wrong? Please link the docs or show the definitions of those bucket functions. — Bergi
– Bergi, Commented Aug 5, 2015 at 2:18
Your delete and list bucket functions call s3 functions which have a callback. So you're functions need to take a callback and call in those callbacks. — bryanmac
– bryanmac, Commented Aug 5, 2015 at 2:47
You should look into Q promises. If you wrap the callback functions with defer and resolve/reject, it flattens out your code alot — bryanmac
– bryanmac, Commented Aug 5, 2015 at 2:47
as an example, we recently converted a class we had to Q and it flattened out alot: github.com/Microsoft/vso-agent/blob/master/src/agent/… — bryanmac
– bryanmac, Commented Aug 5, 2015 at 2:48

Shawn Tsai · Accepted Answer · 2015-08-06 00:54:19Z

function bucketSerialTesting(s3)
{

    Q.fcall(function f1()
    {
         return settingBucket ('testjsbucket20',s3);
    })
    .then(function f2()
    {
        return settingBucket('testjsbucket30', s3);
    }
    )
    .then(function f3()
    {
        return listBucket(s3);
    }
    )
    .then(function f4()
    {
        return deleteBucket('testjsbucket20', s3);
    }
    )
    .then(function f4()
    {
        return deleteBucket('testjsbucket30', s3);
    }
    )
    .then(function f4()
    {
        return listBucket(s3);
    }
    )
    .done();

}



function settingBucket (bucketName, s3){
    var deferred = Q.defer();
    var s3 = new AWS.S3();
    var params = {
          Bucket: bucketName, /* required */
    };
    s3.createBucket(params, function(err, data) {
     if (err) console.log(err, err.stack); // an error occurred
     else     console.log(data);           // successful response
    }).on('success',function(response)
            //put bucket cors for browsers javascript testing
            {
                var params = {
                Bucket: bucketName, /* required */
                CORSConfiguration: {
                CORSRules: [
                  {
                    AllowedHeaders: ['*',],
                    AllowedMethods: ['GET','PUT','DELETE','POST'],
                    AllowedOrigins: ['*','null'],
                    ExposeHeaders:  ['ETag',],
                   },
                ]
                },
                };
                s3.putBucketCors(params, function(err, data) {
                if (err) console.log(err, err.stack); // an error occurred
                else     {
                    console.log(data);
                    console.log('put bucketcors succeed');
                    deferred.resolve(data);
                }
                });
            }).send();
    return deferred.promise;
}




function listBucket(s3)
{
    var deferred = Q.defer();
    s3.listBuckets(function(err, data) {
          if (err) console.log(err, err.stack); // an error occurred
          else     {
            console.log(data);
            console.log('we have '+data.Buckets.length.toString()+' buckets');
            for (var i = data.Buckets.length - 1; i >= 0; i--) {
                console.log(data.Buckets[i].Name);
            };
            deferred.resolve(data);
          }           // successful response
        });
    return deferred.promise;
}

function deleteBucket(bucketName, s3){
    var deferred = Q.defer();
    var params = {
          Bucket: bucketName, /* required */
        };
        s3.listObjectVersions(params, function(err, data) {
          if (err) console.log(err, err.stack); // an error occurred
          else{
              console.log(data);           
              params = {Bucket: bucketName};
                params.Delete = {};
                params.Delete.Objects = [];
//              console.log(data.Versions.length);
                for(j = 0; j < data.Versions.length;++j)
                {
                    params.Delete.Objects.push({Key: data.Versions[j].Key,
                    VersionId: data.Versions[j].VersionId
                    });
                }
                for(j = 0; j < data.DeleteMarkers.length;++j)
                {
                    params.Delete.Objects.push({Key: data.DeleteMarkers[j].Key,
                    VersionId: data.DeleteMarkers[j].VersionId
                    });
                }
                s3.deleteObjects(params, function(err, data) {
                  if (err) {
                      console.log(err, err.stack);
                      console.log('or has no objects');// an error occurred
                  }
                  else     {
                      console.log(data);
                  }
                  params = {Bucket: bucketName};
                  s3.deleteBucket(params, function(err, data) {
                      if (err) console.log(err, err.stack); // an error occurred
                      else     {
                          console.log(data);
                          deferred.resolve(data);
                      }
                    });     
                });

          }
        });
    return deferred.promise;

}

Awesome. glad you figured it out. also don't forget the fail and fin handling. also play with Q with an async function returning data and passing to next (hint s3 should be arg to f3). Lot's of variations and stuff to play with here. beyond the scope of a question.

bryanmac · Accepted Answer · 2015-08-05 10:14:07Z

One option is Q promises which really helps to unwind the callback hell and flatten your code out a bit.

https://github.com/kriskowal/q

The defer, resolve/reject pattern would work well for you.

Here's an example that flattens out async work. It prints a file hello.txt (that exists), waits a second, errors on a file that doesn't exist, jumps to the fail handler and then runs the finally.

var Q = require('q');
var fs = require('fs');
var path = require('path');


// a promise just defers execution of success or failure - must return the promise.  notice how it calls resolve in the callback of the async function.
var wait = function(ms) {
    var defer = Q.defer();

    console.log('waiting ' + ms + ' ms');
    setTimeout(function() {
        defer.resolve();
    }, ms);

    return defer.promise;
}

// you can wrap a node callback function with Q.defer and call reject or resolve
var getFileContents = function(path) {
    var defer = Q.defer();

    fs.readFile(path, function(err, data) {
        if (err) {
            defer.reject(err);
        }
        else {
            // can't pass multiple args back - it returns one obj
            defer.resolve({ path: path, data: data });
        }
    });

    return defer.promise;
}

var ms = 1000;

var printFile = function(filePath) {
    // even function wrapping up work returns a promise so it can be
    // connected with another outer sequence of work
    return wait(ms)
        .then(function() {
            console.log(new Date().toString());
        })
        .then(function() {
            // if you call async that returns promise, you must return it
            return getFileContents(filePath);
        })
        // the resolve data from previous promise is passed as input to next then
        .then(function(contents) {
            console.log(contents.path + ':');
            console.log(contents.data.toString());
        })
}

// sometimes you need to just return a no op based on a conditional check etc...
var noOpPromise = function() {
    console.log('no op');
    return Q(null);
}


printFile(path.join(__dirname, 'hello.txt'))
.then(function() {
    // throw new Error('throwing will jump to fail');
    return wait(1000);
})
.then(function() { 
    return printFile(path.join(__dirname, 'noexist.txt'));
})
.then(function() {
    console.log('should never get here');
})
// can also use .catch
.fail(function(err) {
    console.error('Error: ' + err.message);
})
.fin(function() {
    console.log('done with work');
})

thank you so much, I will also post my edited codes tomorrow.

Collectives™ on Stack Overflow

running multiple async functions sequentially in node.js

2 Answers 2

1 Comment

1 Comment

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

1 Comment

1 Comment

Your Answer

Sign up or log in

Post as a guest

Related