Combining arrays from multiple http.requests into one array

Question

I'm trying to wrap my head around why this loop of https.requests isn't working.

I'm trying to make 5 http requests every minute. Every http GET request calls the function processRequest() as a callback, and every processRequest() returns an array from the request, let's call it resultsArray. I'd like to be able to produce one array or object every minute that is the concatenation of all 5 "resultsArray"s from the 5 http requests, but I'm having trouble figuring out how to declare what variables where. Here is what I have tried:

The CronJob:

var CronJob = require('cron').CronJob;
var tripDataFinal= new Object();

function makeRequest (){
    var urlCounter = 0 ; 
    var feedsList = ["urlA", "urlB", "urlC", "urlD", "urlE"];
    feedsList.forEach((feedToCheck,i)=>{
        options = {
            host: 'host',
            path: feedToCheck 
        }   
        https.request(options, function(response){
            processRequest(response, i, urlCounter, resultsArrayAll);}).on('error', (e) => {
        console.log("Error!"); console.error(e);}).end();   
    })

}
new CronJob('* * * * * *', makeRequest
, null, true, 'America/New_York');

The function processRequest() defined in a separated require() file:

module.exports = {


processRequest : function(response, feedNumber, urlCounter, tripDataFinal) {
        urlCounter++;
        //do stuff to 'response' to produce a result called resultsArray
        resultsArrayAll["array"+urlCounter] = resultsArray;
        console.log(urlCounter);
        if (urlCounter == 5){
            urlCounter = 0 ; 
            return concat(resultsArrayAll.array0
                        , resultsArrayAll.array1
                        , resultsArrayAll.array2
                        , resultsArrayAll.array3
                        , resultsArrayAll.array4);
        }
        else return resultsArrayAll;

    }}

The problem is that 'urlCounter' (which is how I count which request of the 5 I'm on) doesn't seem to be advancing past 1. What (maybe multiple things) am I doing wrong?

Thanks

Have you tried wrapping your http.request in a Promise and then doing Promise.all()? — Sebastian Kaczmarek
– Sebastian Kaczmarek, Commented Oct 11, 2019 at 6:39
I haven't tried that. Would it be possible to do all 5 requests at regular intervals like every minute using the CronJob? — garson
– garson, Commented Oct 11, 2019 at 6:40

Sebastian Kaczmarek · Accepted Answer · 2019-10-11 06:54:22Z

You can wrap your http.request in a Promise and then do Promise.all(), like this:

function makeRequest (){
    var urlCounter = 0 ; 
    var feedsList = ["urlA", "urlB", "urlC", "urlD", "urlE"];
    var requestPromises = feedsList.map(function(url){
        return new Promise(function (resolve, reject){
            options = {
                host: 'host',
                path: feedToCheck 
            }   
            https.request(options, function(response){
                resolve(response);
            }).on('error', (e) => {
                reject(e);
            }).end();   
        });
    });

    Promise.all(requestPromises)
        .then(function(resultsArray){
            console.log('Results:', resultsArray);
        })
        .catch(function(e) { console.error(e); });
}

So here, in resultsArray you will have 5 responses from 5 requests (in the order they were made), and you can do whatever you want with them here, e.x. passing the array to the processRequest() function or something.

iAmADeveloper · Accepted Answer · 2019-10-11 06:50:01Z

You can write your logic here no need to rest the urlCounter inside processRequest

var CronJob = require('cron').CronJob

function makeRequest () {
  var urlCounter = 0
  var feedsList = ['urlA', 'urlB', 'urlC', 'urlD', 'urlE']
  var resultArray = []
  feedsList.forEach((feedToCheck, i) => {
    options = {
      host: 'host',
      path: feedToCheck
    }
    https.request(options, function (response) {
      const result = processRequest(response) // do processing of response and returns a array
      resultArray.concat(result) // concat all results here
    })
  })
}
new CronJob('* * * * * *', makeRequest, null, true, 'America/New_York')

If you want to use concated resultArray you need to use promises here once you done with the all the request you can resolve that promise with resultArray and can use it after it.

Collectives™ on Stack Overflow

Combining arrays from multiple http.requests into one array

2 Answers 2

Comments

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

Comments

Comments

Your Answer

Sign up or log in

Post as a guest

Related