Async and recursive directory scan, for file listing in Nodejs and Expressjs

Question

In this Expressjs route file I'm trying to get (recursively) all the JSON files inside a ./data directory.

Actually I can console.log the file ehere you can see the A Mark, but I can't find the way to send the whole complete bunch of paths to the view once the async stuff finalized.

Some help would be really appreciated.

This is the data ./data structure:

--- dir1
    `-- json1.json
    `-- json2.json
--- dir2
    `-- json3.json
--- dir3

const express = require('express'),
    router = express.Router(),
    fs = require('fs'),
    path = require('path')
    ;

let scan = function (directoryName = './data') {

    return new Promise((resolve, reject) => {

        fs.readdir(directoryName, function (err, files) {
            if (err) reject(err);

            files.map((currentValue, index, arr) => {
                let fullPath = path.join(directoryName, currentValue);

                fs.stat(fullPath, function (err, stat) {
                    if (err) reject(err);

                    if (stat.isDirectory()) {
                        scan(fullPath);
                    } else {
                        console.log(currentValue); <= (A mark)
                        //resolve();
                    }
                });
            });
        });
    })
};


router.get('/', (req, res, next) => {
  scan()
        .then(data => res.render('list', {
            title: 'List',
            data: data
        }))
        .catch(next);
});

module.exports = router;

jfriend00 · Accepted Answer · 2019-06-13 16:51:17Z

9

You can simplify the task a bunch if you promisify the fs functions you're using so that all async logic is promises and then use async/await to help you serialize the flow of control.

Here's one way to do that:

const promisify = require('util').promisify;
const path = require('path');
const fs = require('fs');
const readdirp = promisify(fs.readdir);
const statp = promisify(fs.stat);

async function scan(directoryName = './data', results = []) {
    let files = await readdirp(directoryName);
    for (let f of files) {
        let fullPath = path.join(directoryName, f);
        let stat = await statp(fullPath);
        if (stat.isDirectory()) {
            await scan(fullPath, results);
        } else {
            results.push(fullPath);
        }
    }
    return results;
}

The above code was tested in node v10.14.1.

You could then use that the same way you were:

router.get('/', (req, res, next) => {
  scan().then(data => res.render('list', {
      title: 'List',
      data: data
   })).catch(next);
});

FYI, there is a newer (still experimental) promise-based API for the fs module. You can use that like this:

const path = require('path');
const fsp = require('fs').promises;

async function scan2(directoryName = './data', results = []) {
    let files = await fsp.readdir(directoryName, {withFileTypes: true});
    for (let f of files) {
        let fullPath = path.join(directoryName, f.name);
        if (f.isDirectory()) {
            await scan2(fullPath, results);
        } else {
            results.push(fullPath);
        }
    }
    return results;
}

Note, this new version also uses the new withFileTypes option that saves having to call stat() on every file.

edited Jun 13, 2019 at 16:51

answered Jun 13, 2019 at 16:00

jfriend00

711k104 gold badges1.1k silver badges1k bronze badges

Sign up to request clarification or add additional context in comments.

8 Comments

antzio Over a year ago

A really great and elegant solution, tested with 8.16.0! I still have a doubt: why is the Promisify package necessary for this? Does Promisify do act Readdir and Stat in a sync way? Thank you so much, I have been struggling with this many hours.

jfriend00 Over a year ago

@ant0nio - util.promisify() puts a promise wrapper around any async function that follows the node.js asynchronous calling convention (last argument is a callback function that is called with arguments (err, data). Many people did this manually and then finally node.js added a utility function to make it simpler. If you look at the latest I've added to my answer, the newest versions of node.js actually have a promise version of the fs interface built in which can also be used.

jfriend00 Over a year ago

@ant0nio - promisify() does not make them synchronous, they are still asynchronous, but return promises. The code may look more synchronous because of the use of async and await which works with any promise-returning asynchronous functions.

antzio Over a year ago

I have to say that it would be very helpful for me, to fully understand this subject, if you could give an example of how if would be adding the Promise wrapper in the manual way. If it does not bother you I (and maybe some other nodejs newbies) would appreciate. Thanks!

Jesus is Lord Over a year ago

Here's an answer, based on this answer, that uses fs.promises and ES Module (import) syntax

|

mathertel · Accepted Answer · 2020-10-06 17:40:00Z

The examples above all create one big result array before processing the found entries.

Here is a solution that 'streams' all found file entries of the given directory and sub-directories into an iterator.

Now a filter can be added into the stream to reduce the result to the filter rules. In this examples only the markdown files are accepted.

const fsp = require('fs').promises;
const path = require('path');

// scan the directory recursively and push each filename into the iterator.
async function* scan3(dir) {
  const entries = await fsp.readdir(dir, { withFileTypes: true });
  for (const de of entries) {
    const res = path.resolve(dir, de.name);
    // console.log('>' + res);
    if (de.isDirectory()) {
      yield* scan3(res);
    } else {
      yield res;
    }
  }
}


// get all filenames from the iterator param
// and push each filename with valid extension into the resulting iterator.
async function* filterExt(it, ext) {
  for await (const e of it) {
    if (e.endsWith(ext)) {
      // console.log('>>' + e);
      yield e;
    }
  }
}


async function main() {
  const it_files = scan3('.')
  const it_mdFiles = filterExt(it_files, '.md');

  for await (const f of it_mdFiles) {
    console.log('>>>' + f);
  }
}

main();

console.log("done.");

just enable the console.log lines to see what filename is handled in what stage.

Jesus is Lord · Accepted Answer · 2022-07-08 00:01:44Z

0

Here's an answer, based on this answer, that uses fs.promises and ES Module (import) syntax:

import fs from "fs";
import path from "path";
export async function file_list(directory_name, results = []) {
    let files = await fs.promises.readdir(directory_name, {withFileTypes: true});
    for (let f of files) {
        let full_path = path.join(directory_name, f.name);
        if (f.isDirectory()) {
            await file_list(full_path, results);
        } else {
            results.push(full_path);
        }
    }
    return results;
}

answered Jul 8, 2022 at 0:01

Jesus is Lord

15.4k11 gold badges71 silver badges103 bronze badges

1 Comment

jfriend00 Over a year ago

FYI, this is a copy of the last part of my existing answer with require() changed to import.

Collectives™ on Stack Overflow

Async and recursive directory scan, for file listing in Nodejs and Expressjs

3 Answers 3

8 Comments

Comments

1 Comment

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

3 Answers 3

8 Comments

Comments

1 Comment

Your Answer

Sign up or log in

Post as a guest

Related