node.js loop through array to write files

Question

I've been working through a few others, also this of looping through array to http.get data from a variety of sources. I understand that nodeJS is working asynchronously which is allowing the files to be written empty or with incomplete data, but I can't seem to get past this point.

Problem: calls are made, files are built but the files are always empty

Goal: loop through an array to create files locally from the sites data. Here is what I've got so far:

var file_url = 'http://js.arcgis.com/3.8amd/js/esri/',
    DOWNLOAD_DIR = './esri/',
    esriAMD = [ '_coremap.js', 'arcgis/csv.js'];

function readFile(callback) {
  if (esriAMD.length > 0) {
    var setFile = esriAMD.shift(),
        file_name = url.parse(file_url).pathname.split('/').pop(),
        trial = setFile.split('/').pop(),
        file = fs.createWriteStream(DOWNLOAD_DIR + trial);

    http.get(file_url + esriAMD, function(res) {
      res.on('data', function(data) {
        file.write(data);
        console.log(setFile + ' has been written successfully');
      });
      res.on('end', function(){
        console.log(setFile + ' written, moving on');
        console.log(esriAMD.length);
        readFile(callback);
      });
      //readFile(callback);
    });
  } else {
    callback();
  }
}
readFile(function() {
  console.log("reading finishes");
});

Any insight would really help.

thanks,

try using simpler modules such as request were you can pipe the file directly to the stream. — Gntem
– Gntem, Commented Mar 26, 2014 at 20:23
This "http.get(file_url + esriAMD..." should be " http.get(file_url + file_name...", right? Also, I would create the file inside the callback for the http.get() rather than outside. It should not matter in your case since you don't call readFile() again until res.on('end') but logically it would make more sense inside of it. — Hector Correa
– Hector Correa, Commented Mar 27, 2014 at 0:02
@Phoenix I tried http.request and got a 'socket hang up' error, any thoughts? — atlMapper
– atlMapper, Commented Mar 27, 2014 at 18:29
@HectorCorrea thanks you pointed me in the right direction, I was passing in the wrong variable. I should have been passing in 'setFile' — atlMapper
– atlMapper, Commented Mar 27, 2014 at 18:45

just somebody · Accepted Answer · 2014-03-27 16:42:45Z

0

var esriAMD = [....];
...
function readFile(callback) {
    ...
    http.get(file_url + esriAMD, function(res) {
        ...

concatenating strings with arrays may yield unexpected results.

you want to make sure that

you know what URLs your program is accessing
your program deals with error situations (where the fsck is res.on('error', ...)?)

answered Mar 27, 2014 at 16:42

just somebody

19.4k6 gold badges55 silver badges65 bronze badges

Sign up to request clarification or add additional context in comments.

Comments

atlMapper · Accepted Answer · 2014-03-27 19:05:40Z

Solution: I was passing the wrong variable into the http.get

Working code:

var file_url = 'http://.....',
DOWNLOAD_DIR = './location/';
esriAMD = ['one', 'two', 'three'..0;

function readFile(callback) {
if(esriAMD.length > 0) {
    var setFile = esriAMD.shift(),
    file_name = url.parse(setFile).pathname.split('/').pop(),
    trial = setFile.split('/').pop(),
        file = fs.createWriteStream(DOWNLOAD_DIR + trial);

    http.get(file_url + setFile, function(res){
        res.on('error', function(err){
            console.log(err);
        });
        res.on('data', function(data){
            file.write(data);
            console.log(setFile + ' started');
        });
        res.on('end', function(){ 
            console.log(setFile + ' completed, moving on');
        });
    });
} else {
    callback();
 }
}

Collectives™ on Stack Overflow

node.js loop through array to write files

2 Answers 2

Comments

Comments

Your Answer

Linked

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

Comments

Comments

Your Answer

Sign up or log in

Post as a guest

Linked

Related