MongoDB with Node Index Error

Question

I'm currently using a node.js application to scan Twitter's API based on a set of parameters, and then uploading those JSON objects to a MongoDB database kept on MLab. I have connected to the database without issue, but my code will only upload ONE tweet before crashing. Here is the error message:

(node:62948) UnhandledPromiseRejectionWarning: Unhandled promise rejection (rejection id: 2): BulkWriteError: E11000 duplicate key error index: test-database.test-collection.$_id_ dup key: { : ObjectId('5aecb49e205197f5e4f52e32') }

It seems to have something to do with the keys that I am using in the database? How can I write my code so that I don't have this issue. Here is my program right now:

var Twitter = require("twitter");
var config = require("./config");
const mongoose = require("mongoose");
const MongoClient = require("mongodb");
var twitterClient = new Twitter(config);
const assert = require("assert");

const dbName = "test-database";
const collectionName = "test-collection";
const url = "mongodb://user:[email protected]:13870/test-database";

const param =  {follow: '21111098,958191744683782144,18061669,21111098,18061669,2891210047,1869975300,19394188,4107251,16056306,259459455,21111098,18061669,2891210047,1869975300,19394188,4107251,16056306,259459455,968650362,343041182,5558312,111671288,476256944,378631423,803694179079458816,30354991,224285242,45645232,235217558,20879626,150078976,278124059,102477372,249787913,381577682,15324851,435500714,823302838524739584,20597460,555355209,15745368,229966028,3001665106,2863210809,1397501864,78403308,253252536,47747074,1262099252,1284467173,92186819,169198625,600463589,413160266,1096059529,1095504170,1058520120,328679423,247334603,308794407,216503958,234128524,59969802,10615232,118740781,1383059977,2856787757,75364211,586730005,18632666,18632809,1249982359,339822881,365530059,216881337,3229124078,55677432,816683274076614656,26594419,1068481578,1068540380,19726613,13529632,18137749,3067974778,109071031,278094476,21406834,1129029661,970207298,357606935,236511574,145292853,76456274,456137574,33537967,941000686275387392,555474658,264219447,11650762,16160352,57065141,753693622692970497,21269970,238177562,389554914,11651202,214767677,515822213,16473577,1071402577,323490669,1480852568,2962923040,2987970190,811313565760163844,3145735852,266133081,41363507,109287731,14125897,946549322,361569788,15808765,1603426344,18695134,407039290,1099199839,183062944,60828944,325231436,14140370,17494010,1872999342,72198806,709389393811927041,21157904,213339899,2964174789,22195441,1061029050,460376288,382791093,106733567,43910797,24768753,18915145,240790556,2612307559,7270292,20546536,225921757,27044466,250188760,292495654,122124607,29201047,223166587,171598736,94154021,221162525,26062385,486694111,242555999,770121222,14845376,432895323,3219708271,217543151,81191343,2955485182,978029858,296361085,26533227,76649729,21669223,283130017,73303753,13218102,1648117711,1074480192,23022687,262756641,18170310,88784440,242836537,946946130,172858784,7429102,409719505,293131808,158470209,117501995,35567751,193794406,158890005,234374703,113355380,1074518754,87510313,233737858,291756142,1848942470,202206694,499268312'};

let newTweet = {
    name: "",
    text: "",
    followers: ""
}

MongoClient.connect(url, function(err, client){
  assert.equal(null,err);
  console.log("connected.");

  const db = client.db(dbName);
  const collection = db.collection(collectionName);

  const insertDocument = function(db, callback){

    // THIS IS WHERE I THINK THE PROBLEM IS //
    collection.insert(newTweet)
  }

  twitterClient.stream('statuses/filter',param,function(stream) {
    stream.on('data', function(tweet) {
        newTweet.name = tweet.user.screen_name;
        newTweet.followers = tweet.user.followers_count;
        newTweet.text = (tweet.extended_tweet) ? tweet.extended_tweet.text : tweet.text;

        insertDocument(newTweet, function(){
          db.close();
        });
    });
  });
});

Rajan Panneer Selvam · Accepted Answer · 2018-05-04 19:54:12Z

1

You are closing the database immediately after inserting one tweet.

// stream.on('data', function(tweet) {

insertDocument(newTweet, function(){
   db.close();
});

Instead, close the connection on stream end.

stream.on('end', function() {
    db.close();
});

You are getting the Duplicate key issue, because you declared the newTweet object globally, which shares the same object for every tweet. Declare the tweet object inside the stream.on('data') handler function. i.e.

 stream.on('data', function(tweet) {
    let newTweet = {};
    newTweet.name = tweet.user.screen_name;
    newTweet.followers = tweet.user.followers_count;

answered May 4, 2018 at 19:54

Rajan Panneer Selvam

1,30911 silver badges24 bronze badges

Sign up to request clarification or add additional context in comments.

Collectives™ on Stack Overflow

MongoDB with Node Index Error

1 Answer 1

Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

Comments

Your Answer

Sign up or log in

Post as a guest

Related