Reading multiple files asynchronously in node.js

Question

My application requires to read a folder having multiple files. All files should be fetched asynchronously. Output from all files should be clubbed in a single array. In order to achieve this, below code has been done. (I have used promise). For single file it is working, but for multiple files it is not working. Need your suggestions.

code from files.js

function readFolder(FolderPath){
    return new Promise(function(resolve, reject){
        fs.readdir(FolderPath, (err, files) => {
            if (err) {
                logger.error(`Folder (${FolderPath}) reading Failed :` + err)
                reject(error = "Reading Folder failed")
            } else {
                console.log('resolved')
                resolve(files)
            }
        })
    })
};

function readFile(FilePath){
    return new Promise(function(resolve,reject){
        fs.readFile(FilePath, (err, data) => {
            if (err) {
                reject(error = "Reading file failed") 
            } else {
                console.log('Read File started :'+FilePath)
                var chunk = data.toString();
                var lines = chunk.split('\n');
                var routes = []
                for(var line = 0; line < lines.length; line++){
                    if (lines[line].match(/router.post/)){
                        currentRoute = lines[line].substring(lines[line].indexOf("'")+1 , lines[line].lastIndexOf("'"))
                        route = "/api/angular" + currentRoute
                        routes.push(route)
                    }
                }
                if (routes !== []){
                    console.log('routes for file is :' + routes)
                }

                resolve(routes)
            }
         })
    })
};

function readFiles(FilePaths){
    return new Promise(function(resolve, reject){
        let routesArray = []
        FilePaths.forEach(FilePath => {
            console.log("File Path :"+FilePath)
            readFile(FilePath)
            .then((routes) => {
                console.log('Concatinate')
                routesArray = routesArray.concat(routes)
            })
            .catch((error) => {
                console.log(error)
            })
        })
        console.log(routesArray)
        resolve(routesArray)
    })
}

file name: api.js (Call to promise)


const files = require('./../controlers/files')
files.readFolder(FolderPath)
    .then((filesArray) => {
        var FilePaths = [];
        filesArray.forEach(file => {
            path = "routes/"+file
            FilePaths.push(path)
        })
        console.log(FilePaths)
        return files.readFiles(FilePaths)
    })
    .then((routes) => {
        console.log('routes :', routes)
        res.status(200).send(routes)
    })
    .catch((error) => {
        response.message = "Folder or file Reading failed";
        response.success = false;
        res.status(500).send(response);
    })

Please suggestion where I am wrong.

" it is not working", why? what's happening? Note: When dealing with multiple Promises, like in your readFiles you either deal with one Promise at a time, or use Promise.all to wait for multiple Promises at once - neither of which you are doing - Also, I doubt the code you posted works even for a single file since you resolve(routesArray) before it could even have had anything added to it — Bravo
– Bravo, Commented Oct 17, 2019 at 3:40
Current output is blank array. It is reading the folder and fetch all files. Then for each file when I am trying to fetch the routes, it is giving abrupt response. and keeping final routes array []. — Mohit Aggarwal
– Mohit Aggarwal, Commented Oct 17, 2019 at 11:40

hoangdv · Accepted Answer · 2019-10-17 03:44:51Z

2

You are "wrong" when use .forEach with Promise syntax (in readFiles function). .forEach is a "callback" style function, it will not work as you expectly with Promise.

You need wait until all files have been done, my suggestion is using Array.map and Promise.all:

function readFiles(FilePaths) {
  return new Promise(function (resolve, reject) {
    let routesArray = []
    const promises = FilePaths.map(FilePath => { // get back an array of promises
      console.log("File Path :" + FilePath)
      return readFile(FilePath)
        .then((routes) => {
          console.log('Concatinate')
          routesArray.push(...routes) // I like .push function
        })
        .catch((error) => {
          console.log(error)
        })
    });
    Promise.all(promises)
      .then(() => { // all done!
        console.log(routesArray)
        resolve(routesArray)
      })
  })
}

answered Oct 17, 2019 at 3:44

hoangdv

16.3k4 gold badges31 silver badges54 bronze badges

Sign up to request clarification or add additional context in comments.

1 Comment

Mohit Aggarwal Over a year ago

Thanks ! Combination of Array.map and Promise.all worked for me.

Dženis H. · Accepted Answer · 2019-10-17 17:01:12Z

Since we're talking about multiple files, may I suggest creating areadAsync function, and passing it some options ('utf8' here) and a callback?

function readAsync(file, callback) {
  fs.readFile(file, 'utf8', callback);
}

After that, we could map over asynchronously and try to read the content, like this:

// we're passing our files, newly created function, and a callback

async.map(files, readAsync, (err, res) => { 
        // res = ['file 1 content', 'file 2 content', ...]
    });

I believe you can shorten the whole process to this:

// Let's put everything inside of one function:

function testAsync() {
    const files = ['file1.json', 'file2.json', 'file3.json'];
    async.map(files, fs.readFile, function (err, data) {
        for(let file of files) {
            console.log( file.toString() );
        }
    });
}

Here's example code using Promises:

function readFromFile(file) {
    return new Promise((resolve, reject) => {
        fs.readFile(file, (err, data) => {
            if (err) {
                console.log(err);
                reject(err);
            }
            else {
                resolve(JSON.parse(data));
            }
        });
    });
}

// First, an array of promises are built.
// Each Promise reads the file, then calls resolve with the result.
// This array is passed to Promise.all(), which then calls the callback, 
// passing the array of results in the same order.

const promises = [
    readFromFile('result1.json'),
    readFromFile('result2.json')
    // ETC ...
];

Promise.all(promises)
 .then(result => {
    baseList = result[0];
    currentList = result[1];
    // do more stuff
});

Collectives™ on Stack Overflow

Reading multiple files asynchronously in node.js

2 Answers 2

1 Comment

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

1 Comment

Comments

Your Answer

Sign up or log in

Post as a guest

Related