Node.JS JSON.parse UTF-8 issue

Question

I've built a small script that allow me to get keywords suggestion from Google search API.

The main problem is if the response contain special characters (like à é ù etc.) : my application return me unreadable keywords like that : ï¿½,ï¿½a,ï¿½a va,ï¿½ majuscule,ï¿½a marche,ï¿½,ï¿½a y est,ï¿½a film,ï¿½gag,ï¿½a il est revenu,ï¿½a va de soi,,[object Object]

Here's my Node.js Script :

var express = require('express');
var request = require('request');
var app = express();
app.get('/sug', function (req, res) {
    var KW = req.query.KW ;
    console.time("Délai");
    var url = "http://clients1.google.fr/complete/search?hl=fr&q=" + KW + "&json=t&client=hp";
    request(url, function (err, resp, body) {
        body = JSON.parse(body);
        res.end(body.toString());
        console.timeEnd("Délai");
    });
});
app.listen(1337);
console.log('Serveur : OK');

The call is easy to make, just type http://localhost:1337/sug?KW=ç in your browser.

Do you know how to solve this and get the utf-8 working ?

The URL you posted is a network local URL, and therefore not accessible for anyone not on your network (i.e. us). — Tiddo
– Tiddo, Commented Dec 21, 2015 at 11:34
problem with clients1.google.fr response. It's encoded with "ISO-8859-1" and it returns ["ç",["�","�a","� encoding","�in takvimi","�eyrek alt�n fiyat�","��plak k�zlar","�a��atay ulusoy","��plak kad�nlar","�ukurova �1\u20444niversitesi","�ocuk nas�l yap�l�r"],[],{"google:suggesttype":["QUERY","QUERY","QUERY","QUERY","QUERY","QUERY","QUERY","QUERY","QUERY","QUERY"]}] But after convert into UTF-8 we get that strange symbols ¿½,ï¿½a — vmkcom
– vmkcom, Commented Dec 21, 2015 at 12:37
@vmkcom So why this work in this URL : clients1.google.fr/complete/… ?? — Arnaud Boyer
– Arnaud Boyer, Commented Dec 21, 2015 at 12:41
@Tiddo Yes, you must to install node.js and run my script to see it on your browser ;) — Arnaud Boyer
– Arnaud Boyer, Commented Dec 21, 2015 at 12:42
Because your browser is honouring the Content-Type HTTP response header and is interpreting the encoding correctly. — deceze
– deceze ♦, Commented Dec 21, 2015 at 12:42

Shanoor · Accepted Answer · 2015-12-21 12:46:24Z

1

Like vmkcom said, it's because the response is using ISO-8859-1, you have to manually convert to utf-8. iconv package can help you with that:

var request = require('request');
var iconv = require('iconv');

var KW = 'ç' ;
console.time("Délai");
var url = "http://clients1.google.fr/complete/search?hl=fr&q=" + KW + "&json=t&client=hp";

var options = {
    url: url,
    encoding: null // << set encoding to null so request don't try to force utf-8
};

var ic = new iconv.Iconv('iso-8859-1', 'utf-8');

request(options, function (err, resp, body) {
    // body is a Buffer not a string, convert to utf-8 buffer then to utf-8 string
    body = ic.convert(body).toString('utf-8');
    console.log(body);
    console.timeEnd("Délai");
});

answered Dec 21, 2015 at 12:46

Shanoor

13.7k2 gold badges32 silver badges41 bronze badges

Sign up to request clarification or add additional context in comments.

Comments

Arnaud Boyer · Accepted Answer · 2015-12-22 10:41:38Z

Thanks for all the help. So i did this code using your feedbacks :

var express = require('express');
var request = require('request');
var iconv = require('iconv');
var app = express();
app.listen(1337);
console.log('Serveur : OK');
app.get('/sug', function (req, res) {
    var KW = req.query.KW;
    console.time("Délai");
    var url = "http://clients1.google.fr/complete/search?hl=fr&q=" + KW + "&json=t&client=hp";
    var options = {
        url: url,
        encoding: null // << set encoding to null so request don't try to force utf-8
    };
    var ic = new iconv.Iconv('iso-8859-1', 'utf-8');
    request(options, function (err, resp, body) {
        res.set({ 'content-type': 'application/json; charset=utf-8' });
        body = ic.convert(body).toString('utf-8');
        body = JSON.parse(body);
        res.end(body.toString());
        console.timeEnd("Délai");
    });
});

It work great !

http://localhost:1337/sug?KW=%C3%A7a

ça,ça,ça va,ça marche,ça y est,ça film,ça il est revenu,ça va de soi,ça te va,ça s'est bien passé,ça m'énerve,,[object Object]

developerjavad · Accepted Answer · 2020-11-19 16:13:28Z

0

response set
response.set({ 'content-type': 'application/json; charset=utf-8' });

app.use('/reverse',function (requsting,response) { request.get({ url: url, json: true, headers: {'User-Agent': 'request'} }, (err, res, data) => { if (err) { console.log('Error:', err); } else if (res.statusCode !== 200) {

        response.end(JSON.stringify({status : 'error'}));

    } else {
        response.set({ 'content-type': 'application/json; charset=utf-8' });

        response.end(JSON.stringify({status : 'ok','api' : 'website : https://homeandroid.ir','json':data}));

    }
});

});

answered Nov 19, 2020 at 16:13

developerjavad

3421 gold badge2 silver badges11 bronze badges

Collectives™ on Stack Overflow

Node.JS JSON.parse UTF-8 issue

3 Answers 3

Comments

Comments

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

3 Answers 3

Comments

Comments

Comments

Your Answer

Sign up or log in

Post as a guest

Related