Nodejs Mongodb find performance issue

Question

I just recently put hands on mongodb, by that time I have been having many issues. I'm creating a card game, so I thought mongodb would be the best option for this.

Now I'm trying to find all my deck,hand and table card, the problem is that it takes so long to execute, even if collection is pretty small.

e.g

{"_id":"5bd3323b1431aa01eed604a8","userID":1,"handData":[{"id":"MLTOG","suit":"Spades","value":5,"face":"5","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"3ZgGQ","suit":"Spades","value":3,"face":"3","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"zdgyN","suit":"Clubs","value":2,"face":"2","userHide":false,"globalHide":false,"hasSuperPower":true},{"id":"oy4CS","suit":"Spades","value":4,"face":"4","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"PgTxA","suit":"Diamonds","value":3,"face":"3","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"zlx6Y","suit":"Diamonds","value":4,"face":"4","userHide":false,"globalHide":false,"hasSuperPower":false}]}

{"_id":"5bd3315f11716c01d3a5a4a4","gameID":"aasd12","deckData":[{"id":"rPd2D","suit":"Diamonds","value":7,"face":"7","userHide":false,"globalHide":false,"hasSuperPower":true},{"id":"ZslMO","suit":"Hearts","value":14,"face":"Ace","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"vJ4j2","suit":"Hearts","value":13,"face":"King","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"sQWuO","suit":"Clubs","value":12,"face":"Queen","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"SV6wb","suit":"Clubs","value":4,"face":"4","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"XGut7","suit":"Hearts","value":4,"face":"4","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"jmiM8","suit":"Hearts","value":3,"face":"3","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"BqQgl","suit":"Hearts","value":7,"face":"7","userHide":false,"globalHide":false,"hasSuperPower":true},{"id":"1sDky","suit":"Hearts","value":6,"face":"6","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"PWTot","suit":"Spades","value":12,"face":"Queen","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"ctBqJ","suit":"Spades","value":10,"face":"10","userHide":false,"globalHide":false,"hasSuperPower":true},{"id":"g7MX1","suit":"Clubs","value":7,"face":"7","userHide":false,"globalHide":false,"hasSuperPower":true},{"id":"voGO8","suit":"Hearts","value":5,"face":"5","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"SeOwQ","suit":"Spades","value":14,"face":"Ace","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"67eKt","suit":"Clubs","value":11,"face":"Jack","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"rBfas","suit":"Spades","value":13,"face":"King","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"xD5X6","suit":"Clubs","value":14,"face":"Ace","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"91yRB","suit":"Diamonds","value":11,"face":"Jack","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"TMbbQ","suit":"Spades","value":7,"face":"7","userHide":false,"globalHide":false,"hasSuperPower":true},{"id":"6NQvl","suit":"Diamonds","value":5,"face":"5","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"0pMrB","suit":"Clubs","value":10,"face":"10","userHide":false,"globalHide":false,"hasSuperPower":true},{"id":"zIMFm","suit":"Hearts","value":12,"face":"Queen","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"KeQNt","suit":"Hearts","value":8,"face":"8","userHide":false,"globalHide":false,"hasSuperPower":true},{"id":"KSxvM","suit":"Clubs","value":8,"face":"8","userHide":false,"globalHide":false,"hasSuperPower":true},{"id":"MaNEM","suit":"Clubs","value":5,"face":"5","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"jU2h3","suit":"Hearts","value":10,"face":"10","userHide":false,"globalHide":false,"hasSuperPower":true},{"id":"VqKOQ","suit":"Diamonds","value":9,"face":"9","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"S0dz2","suit":"Spades","value":2,"face":"2","userHide":false,"globalHide":false,"hasSuperPower":true},{"id":"ehIbj","suit":"Spades","value":6,"face":"6","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"VsDTD","suit":"Diamonds","value":13,"face":"King","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"tmzY9","suit":"Clubs","value":3,"face":"3","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"eSQ8n","suit":"Hearts","value":2,"face":"2","userHide":false,"globalHide":false,"hasSuperPower":true},{"id":"y3fEo","suit":"Clubs","value":9,"face":"9","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"m5EBi","suit":"Diamonds","value":14,"face":"Ace","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"4IFqQ","suit":"Diamonds","value":10,"face":"10","userHide":false,"globalHide":false,"hasSuperPower":true},{"id":"KlGhF","suit":"Spades","value":8,"face":"8","userHide":false,"globalHide":false,"hasSuperPower":true},{"id":"wsLfO","suit":"Diamonds","value":12,"face":"Queen","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"9O44l","suit":"Hearts","value":11,"face":"Jack","userHide":false,"globalHide":false,"hasSuperPower":false},{"id":"H0r9u","suit":"Diamonds","value":6,"face":"6","userHide":false,"globalHide":false,"hasSuperPower":false}]}

The fetch functions are like this, of course the mongoClient connection is not way to go, but im just using this function for testing.

const getDeckCards =  (gameid) =>{
    return new Promise ((resolve,reject) => {
      console.time("deck");
    MongoClient.connect(url, { useNewUrlParser: true }, function (err, client) {
    const db = client.db(dbName);
    const collection = db.collection("deck");
    resolve(collection.find({ gameID: gameid }).project({ _id: 0, gameID: 0 }).toArray());
    });
    console.timeEnd("deck");
  });
  }

  const getHandCards =  (userid) =>{
    return new Promise ((resolve,reject) => {
      console.time("hand");
    MongoClient.connect(url, { useNewUrlParser: true }, function (err, client) {
      const db = client.db(dbName);
    const collection = db.collection("hand");
    resolve(collection.find({ userID: userid }).project({ _id: 0, userID: 0 }).toArray());
    });
    console.timeEnd("hand");
  });
  }

  const getTableCards =  (gameid) =>{

    return new Promise ((resolve,reject) => {
      console.time("table");
    MongoClient.connect(url, { useNewUrlParser: true }, function (err, client) {
      const db = client.db(dbName);
    const collection = db.collection("table");
    resolve(collection.find({ gameID: gameid }).project({ _id: 0, gameID: 0 }).toArray());
    });
    console.timeEnd("table");
  });
  }

I try to get all promises and then return them as array.

const initGame = (async() => {
    console.time("init");
   let [deck,hand,table] = await Promise.all([
      getDeckCards("aasd12"),
      getHandCards(1),
      getTableCards("aasd12")
    ])

    console.timeEnd("init");
    return [deck[0],hand[0],table[0]];
  });

It shows that the deck,hand,table functions were pretty fast, THE result that I'm hoping to get, but the whole result takes 1 second what is incredibly slow for such small querys

deck: 12.235ms
hand: 1.977ms
table: 0.534ms
init: 1058.646ms

What I'm doing wrong? Is it the promise all function that takes soo long to execute or is it problem with my Mongodb query?

The main issue is that you're creating a new database connection for each function call, which is very inefficient. Create a connection (MongoClient instance) once, and reuse it. Also, you're not timing the correct thing (console.timeEnd should be added after the call the resolve, but in the same scope). — robertklep
– robertklep, Commented Nov 9, 2018 at 14:34
Creating new connection every time and calling toArray are both expensive. What is the amount of records we are talking about here? — Akrion
– Akrion, Commented Nov 9, 2018 at 23:12

Manuel Spigolon · Accepted Answer · 2018-11-09 17:29:05Z

1

The problem is that you create a new connection to mongo every time.

The solution is to create a single connection and maintain it alive and close it when is no more needed (when the application is stopping).

This example code will sure have a good impact on your performance:

class GameManager {
  constructor() {
    this.mongoClient = null;
  }

  async connect(dbName) {
    this.mongoClient = await MongoClient.connect(url, { useNewUrlParser: true });
    this.db = this.mongoClient.db(dbName);
  }

  getDeckCards(gameId) {
    if (!this.mongoClient) {
      return Promise.reject();
    }

    return this.db.collection("deck")
      .find({ gameID: gameId })
      .project({ _id: 0, gameID: 0 })
      .toArray();
  }
}

const initGame = () => {
  console.time("init");
  const gameManager = new GameManager();
  return gameManager.connect(dbName)
    .then(() => {
      return Promise.all([
        getDeckCards("aasd12"),
        getHandCards(1),
        getTableCards("aasd12")
      ])
    })
    .then(() => {
      console.timeEnd("init");
    });
};

Remember also to create INDEXes on mongo on those fields you search for (for example the gameID in this case). With this management, you could access data in a faster way.

answered Nov 9, 2018 at 17:29

Manuel Spigolon

13.4k7 gold badges60 silver badges95 bronze badges

Sign up to request clarification or add additional context in comments.

2 Comments

Godhaze Over a year ago

Thank you, for your answer, really appreciate your time for this. I tried your code, but oddly the result are the same: init time = 1060.204ms, when trying to add index e.g this.db.collection("deck").createIndex({"deckData":1}); It takes even longer init time = 2078.974ms. I also thought it maybe a problem with using mongod in my ubuntu subsystem, but when installing directly on my Windows 10, it wasen't the case

Manuel Spigolon Over a year ago

The init time is always slow because you have to establish the connection to the DB. Have you checked the time in the query? Because my expectation is that the queryes (getDeckCards, getHandCards, getTableCards) are in order of some millisec. Another way to verify the latency is use bubbleprof (clinicjs.org/bubbleprof) that is quite simple, just run clinic bubbleprof - node yourscript.js

Collectives™ on Stack Overflow

Nodejs Mongodb find performance issue

1 Answer 1

2 Comments

Your Answer

Hot Network Questions

Collectives™ on Stack Overflow

1 Answer 1

2 Comments

Your Answer

Sign up or log in

Post as a guest

Related