Stdin read fails on some input

Question

#!/usr/bin/env node

function stdinReadSync() {
   var b = new Buffer(1024);
   var data = '';

   while (true) {
      var n = require('fs').readSync(process.stdin.fd, b, 0, b.length);
      if (!n) break;
      data += b.toString(null, 0, n);
   }
   return data;
}

var s = stdinReadSync();
console.log(s.length);

The above code (taken from Stackoverflow) works just fine if you feed it with echo, cat, ls, but will fail with curl output.

$ echo abc | ./test.js
4

$ ls | ./test.js
1056

$ cat 1.txt | ./test.js
78

$ curl -si wikipedia.org | ./test.js
fs.js:725
  var r = binding.read(fd, buffer, offset, length, position);
                  ^

Error: EAGAIN: resource temporarily unavailable, read
    at Error (native)
    at Object.fs.readSync (fs.js:725:19)
    at stdinReadSync (/home/ya/2up/api/stdinrd.js:8:29)
    at Object.<anonymous> (/home/ya/2up/api/stdinrd.js:15:9)
    at Module._compile (module.js:541:32)
    at Object.Module._extensions..js (module.js:550:10)
    at Module.load (module.js:456:32)
    at tryModuleLoad (module.js:415:12)
    at Function.Module._load (module.js:407:3)
    at Function.Module.runMain (module.js:575:10)
(23) Failed writing body

Why? How to fix?

robertklep · Accepted Answer · 2016-11-01 15:27:17Z

7

It's a bit of a hack, but this seems to work:

var n = require('fs').readSync(0, b, 0, b.length);

I think (pure conjecture) that process.stdin.fd is a getter that, when referenced, will put stdin in non-blocking mode (which is causing the error). When you use the file descriptor directly, you work around that.

answered Nov 1, 2016 at 15:27

robertklep

205k38 gold badges421 silver badges413 bronze badges

Sign up to request clarification or add additional context in comments.

Comments

Paul Rumkin · Accepted Answer · 2016-11-10 16:33:23Z

3

It's a problem of synchronous reading from stdin and as I see there is no solution for it and it wouldn't fixed, because process.stdin.fd is not a part of a public API and should not be used in any way. Better to use promisified version to avoid this errors and read from stdin:

function streamToPromise(stream) {
    return new Promise((resolve, reject) => {
        let chunks = [];

        function onData(chunk) {
            chunks.push(chunk);
        };

        function onEnd() {
            unbind();
            resolve(Buffer.concat(chunks));
        };

        function onError(error) {
            unbind();
            reject(error);
        };

        function unbind() {
            stream.removeListener('data', onData);
            stream.removeListener('end', onEnd);
            stream.removeListener('error', onError);
        }

        stream.on('data', onData);
        stream.on('end', onEnd);
        stream.on('error', onError);
    });
}

streamToPromise(process.stdin).then((input) => {
    // Process input
});

edited Nov 10, 2016 at 16:33

answered Nov 1, 2016 at 15:16

Paul Rumkin

6,9052 gold badges29 silver badges36 bronze badges

1 Comment

Felipe Balduino Cassar Over a year ago

This worked great! I was using readSync before and I had several problems with large inputs. With this stream I successfully tested a 40MB file as input.

ten0s · Accepted Answer · 2022-12-02 17:42:55Z

EAGAIN means the resource is TEMPORARILY unavailable and the request should be retried

#!/usr/bin/env node

const fs = require('fs')

function stdinReadSync() {
    const b = Buffer.alloc(1024)
    let data = ''

    while (true) {
        let n = 0

        // Read while EAGAIN
        while (true) {
            try {
                n = fs.readSync(process.stdin.fd, b, 0, b.length)
                break
            } catch (e) {
                if (e.code === 'EAGAIN') {
                    // Sleep 100ms
                    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 100)
                    continue
                }
                throw e
            }
        }

        if (!n) break
        data += b.toString('utf8', 0, n)
    }

    return data
}

const s = stdinReadSync()
console.log(s.length)

itsadok · Accepted Answer · 2023-04-06 09:50:29Z

2

As mention in the github link shared by Paul Rumkin, the simplest solution would be to open "/dev/stdin" rather than process.fd.stdin:

$ (sleep 1; echo hi) | node -e 'console.log(require("fs").readFileSync(process.stdin.fd).toString())'
...
Error: EAGAIN: resource temporarily unavailable, read
...
$ (sleep 1; echo hi) | node -e 'console.log(require("fs").readFileSync("/dev/stdin").toString())'
hi

Obviously this is not portable and won't work on Windows.

answered Apr 6, 2023 at 9:50

itsadok

29.5k32 gold badges133 silver badges176 bronze badges

Collectives™ on Stack Overflow

Stdin read fails on some input

4 Answers 4

Comments

1 Comment

Comments

Comments

Your Answer

Linked

Hot Network Questions

Collectives™ on Stack Overflow

4 Answers 4

Comments

1 Comment

Comments

Comments

Your Answer

Sign up or log in

Post as a guest

Linked

Related