Stream Rest API HTTP Response in Node.js & Express

Question

I am creating a Rest API in Node.js and Express. It connects with remote HANA database & execute one query. Now I want to stream HTTP response so that I can send it to browser into chunks, rather than sending it completely since it's a very large data.

I tried something which is giving me no output. I don't know the reason. If I send the complete response to browser using response.send (data), it's working fine. But streaming is now working.

I have added code snippet below.

const express = require("express");
const APP = express();
const HANA_DB = require('hdb');
const BODY_PARSER = require("body-parser");

start();

function start() {
    startServer();
    initializeExpress();
    APP.get("/data", function(request, response) {
        var connection = HANA_DB.createClient({
            host     : "hostname",
            port     : "port",
            user     : "username",
            password : "password"
        });
        connection.on('error', function (error) {
            console.log("Error in database connection...");
        });
        connection.connect(function (error) {
            if (error) {
                console.log("Error in database connection...");
                return;
            }
            var query = "SELECT * FROM TableName";
            connection.exec(query, function(error, result) {
                if(error) {
                    response.send("Getting error while fetching result...");
                    return;
                }
                //response.send(data);
                var datalength = 0;
                request.on('data', function(chunk) {
                    datalength += chunk.length;
                    console.log("DATA EVENT: " + datalength);
                    response.send(datalength);
                })
                .on('end', function() {
                    console.log("END EVENT: " + datalength);
                    response.send(datalength);
                });
            });
        });     
    });
};

function initializeExpress() {
    APP.all('/*', function(request, response, next) {
        response.header("Access-Control-Allow-Origin", "*");
        response.header("Access-Control-Allow-Headers", "X-Requested-With");
        response.header('Access-Control-Allow-Methods', 'PUT, GET, POST, DELETE, OPTIONS');
        response.header('Access-Control-Allow-Headers', 'Content-Type');
        next();
    });
    APP.use(BODY_PARSER.json());
    APP.use(BODY_PARSER.urlencoded({ extended: true }));
};

function startServer(config) {
    var server = APP.listen("8081", function(error) {
        if(error) {
            console.log("Unable to connect to 127.0.0.1:8081");
            return;
        }
        console.log("Server is listening at - 127.0.0.1:8081");
    });
};

Mohd Shahid · Accepted Answer · 2019-02-28 05:43:33Z

2

I think the logic you are using is wrong for the streaming the data.

Use res.write instead of res.send and you also have to read streaming data from your database instead of one time connection.exec

I am giving you an example code where you will get some idea about streaming data in Expressjs.

var http = require( 'http' );

http.createServer( function ( req, res ) {
res.writeHead( 200, {
    'Content-Type': 'text/plain; charset=utf-8',
    'Transfer-Encoding': 'chunked',
    'X-Content-Type-Options': 'nosniff'
} );
res.write( 'Beginning\n' );
var count = 10;
var io = setInterval( function () {
    res.write( 'Doing ' + count.toString() + '\n' );
    count--;
    if ( count === 0 ) {
        res.end( 'Finished\n' );
        clearInterval( io );
    }
}, 1000 );
} ).listen( 8000 );

answered Feb 28, 2019 at 5:43

Mohd Shahid

1,6163 gold badges35 silver badges71 bronze badges

Sign up to request clarification or add additional context in comments.

Comments

Lorenz Meyer · Accepted Answer · 2017-03-03 07:34:46Z

0

The problem is here request.on('data',. request refers to the browser request.

You cannot use streaming with .exec(), because the callback function of exec is called with the rows as parameters.

To use streaming, use the .execute() method, which passes a resultset to the callback function.

I never used hdb, so I cannot give the code to use.

edited Mar 3, 2017 at 7:34

answered Mar 3, 2017 at 6:30

Lorenz Meyer

20.1k23 gold badges83 silver badges130 bronze badges

4 Comments

Devesh M Over a year ago

Do you mean to say to that i should set data & end event with result returning by query ? I tried it, but it's saying that result.on is not a function. How it would be possible, because result is not a writable stream.

Devesh M Over a year ago

Thanks for your solution. I tried .execute method and i it's working for me. But now i am facing issues with sending chunk data to browser in data event. I am trying response.send(chunk) which giving error "Can't set headers after they are sent". It seems response.send(chunk),i can execute only one time. If it is, then how i will send chunks to browser in data event.

Lorenz Meyer Over a year ago

@DeveshM As I said, I never used hdb, so I'm sorry I cannot help you further.

Devesh M Over a year ago

Sorry to bother you again. But i am not getting any clue on what i am looking for. Now I am able to stream results for both Hana & Oracle. But i am not able to send chunk data to browser back in data event. Can you guide me whether is it worth to send chunk data every time or should i send it to browser after loading complete data which i gets in end event of stream ?

Collectives™ on Stack Overflow

Stream Rest API HTTP Response in Node.js & Express

2 Answers 2

Comments

4 Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

2 Answers 2

Comments

4 Comments

Your Answer

Sign up or log in

Post as a guest

Related