Recursive java script function - out of control

Question

The code below works. The code calls an API to get historical trades (100 trades each time pér pull). Because there is an limit - how many and how often im allowed to call the API - the structure is like recursive.

The flow is like this:

Get the current MAX tradeId - which is stored in the DB.
Now make a new PULL with startIndex = MaxId and a length of 100 (to pull 100 new trades).
FIRST when the callback function is called the main code continues and 100 new trades are pulled... ect. ect.

So...

The code SHOULD behave like - Psydo-code

var maxId = GetMaxIdFromDB();

Pull(maxId, 100, callback);

function callback(){

... do different stuff..
maxId += 100;

Pull(maxId, 100, callback);

}

The strange thing and my question is: How can the API function "getProductTrades " be called more than one time - where my cursor variable contains the SAME value - when it is incremented with 100 (or the number of valid data elements each time).

I'm talking/ref. especially to the following lines:

wl.debug("getProductTrades - cursor: " + cursor + " Limit: " + limit);
                publicClient.getProductTrades({'after': cursor, 'limit': limit}, callback);

The insertQuery.InsertMatchMsgArrayToDB(allData); method calls another DB method which returns a promise.

You can see a screenshot of the issue here: http://screencast.com/t/DH8rz3UxnyZ

The real code is here:

pullTradesBetween: function (minTradeId, maxTradeId) {

            var wl = new WinLog();
            var tradeCounter = 0;

            try {
                var WebSocketEmit = new WSemitter();
                var startTime = new Date().toLocaleString();
                var executeTradePullAgain = null;

                wl.debug("REST API START: " + startTime);

                var cursor;
                var incrementedCursorWith = 0;

                if ((maxTradeId - minTradeId) < 100) {
                    cursor = maxTradeId + 1;
                }
                else
                    cursor = minTradeId + 100;

                var callback = function (err, response, data) {

                    if (executeTradePullAgain !== null)
                        clearTimeout(executeTradePullAgain);

                    if (err)
                        wl.info("Err: " + err);

                    var validData = [];
                    incrementedCursorWith = 0;

                    if (response == null)
                        wl.info("RESPONSE ER NULL");

                    if (data !== null) {
                        for (var i = data.length - 1; i >= 0; i--) {
                            var obj = data[i];

                            var tradeId = parseInt(obj.trade_id);

                            if (obj !== null && (minTradeId <= tradeId && tradeId <= maxTradeId)) {
                                validData.push(data[i]);
                            }
                        }

                        if (validData.length == 0) {
                            wl.debug("Contains 0 elements!");
                        }
                        else {
                            cursor = cursor + validData.length;
                            incrementedCursorWith = validData.length;
                            insertDataToDB(validData);
                        }
                    }
                    else
                        wl.debug("DATA IS NULL!");

                    wl.debug("cursor: " + cursor + " maxTradeId: " + maxTradeId);

                    var diffToMax = maxTradeId - (cursor - incrementedCursorWith);

                    if (diffToMax >= 100)
                        pullTrades(cursor, 100); // 100 is default
                    else if (diffToMax >= 0)
                        pullTrades(maxTradeId + 1, diffToMax + 1); // X = Only the last trades in the given series of trades
                    else {
                        wl.info("REST API START: " + startTime + " REST API DONE: " + new Date().toLocaleString());

                        WebSocketEmit.syncHistoricalDataDone();
                    }
                };

                function pullTrades(cursor, limit) {

                    tradeCounter += limit;

                    if(tradeCounter % 10000 == 0){
                        wl.info('Downloaded: ' + tradeCounter + ' trades via REST API (Total: ' + cursor + ')');
                    }

                    pullTradesAgainIfServerDoesNotRespond(cursor, limit);

                    wl.debug("getProductTrades - cursor: " + cursor + " Limit: " + limit);
                    publicClient.getProductTrades({'after': cursor, 'limit': limit}, callback);
                }

                function pullTradesAgainIfServerDoesNotRespond(cursor, limit) {

                    executeTradePullAgain = setTimeout(function () {
                        wl.debug('pullTradesAgainIfServerDoesNotRespond called!');
                        pullTrades(cursor, limit);
                    }, 30000);
                }

                // SAVE DATA IN DB!
                function insertDataToDB(allData) {
                    insertQuery.InsertMatchMsgArrayToDB(allData);
                }

                wl.debug("pull trades: " + cursor);
                pullTrades(cursor, 100);
            }
            catch(err){
                wl.info('pullTradesBetween: ' + err);
            } }};

Are you sure publicClient.getProductTrades() calls the callback only once every time? — Johannes Merz
– Johannes Merz, Commented Aug 24, 2016 at 17:14
The API DOC doesn't say any thing about that - but i would find it rather strange - if the API return 5 x 20 objects - when i request 100 ...haha... I just added some more logging to test ... and i look like it only and always return, calls, returns, calls... look here: screencast.com/t/YkGvgfBcOMqb — PabloDK
– PabloDK, Commented Aug 24, 2016 at 21:58
This might be a naive point of view here... but what excactly does guarantee maxId - minId equals 100? Trade entries could have been deleted, this does not really look like a safe thing to assume — a-d
– a-d, Commented Aug 24, 2016 at 23:09
i pull/request 100 - the API will allway return the given number of items... until no more item exist to pull... the data contains a seq.-no which the client use to ensure - that the trades comes in the correct order - and nothing is missing... the client needs to implement error-handling for this... like i did.. look a "validData" ... — PabloDK
– PabloDK, Commented Aug 25, 2016 at 0:18

Johannes Merz · Accepted Answer · 2016-08-25 07:57:00Z

1

It happens when you get no data out of getProductionTrades. If the data returned is null, you will never reach the lines

cursor = cursor + validData.length;
incrementedCursorWith = validData.length;

but you still call

pullTrades(cursor, 100);

at the end. I don't know if it's intended or an actual error so i leave the solution (should be trivial now) up to you.

answered Aug 25, 2016 at 7:57

Johannes Merz

3,34020 silver badges35 bronze badges

Sign up to request clarification or add additional context in comments.

Comments

Aikon Mogwai · Accepted Answer · 2016-08-24 23:04:36Z

I try to simplify your code

pullTradesBetween: function (minTradeId, maxTradeId) {
    var WebSocketEmit = new WSemitter(); // try-catch ?
    var curr = (maxTradeId - minTradeId < 100) ? maxTradeId + 1 : minTradeId + 100;

    // function always return data or infinite error-loop
    function getProductTrades (after, limit, callback) {
        // try-catch ?
        publicClient.getProductTrades ({after, limit}, function(err, data) {
            if (err) {
                console.log(err);
                return getTrades(after, limit, callback);
            }

            callback(null, data);
        });
    }

    function onDataReady (err, data) {
        if (err)
            throw new Error('Impossible!');

        if (!data || !(data instanceof Array))  
            return ... smth on empty data ...

        var validData = data.filter(function(obj) {
            return obj && 
                minTradeId <= parseInt(obj.trade_id) &&
                parseInt(obj.trade_id) <= maxTradeId;
        }).reverse();

        if (validData.length == 0)
            return ... smth on empty data ...

        insertDataToDB(validData);

        curr += validData.length; // maybe +-1 

        var remaining = maxTradeId - curr;
        if (remainig == 0) {
            console.log('Done');
            // try-catch ?
            WebSocketEmit.syncHistoricalDataDone();
        }

        return (remaining >= 100) ? 
            getProductTrades(curr, 100, onDataReady) : 
            getProductTrades(maxTradeId + 1, remaining + 1, onDataReady); // ??
    }

    getProductTrades(curr, 100, onDataReady);
}

Collectives™ on Stack Overflow

Recursive java script function - out of control

2 Answers 2

Comments

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

Comments

Comments

Your Answer

Sign up or log in

Post as a guest

Related