Migrating Data in Mongo

Question

I have the follow script to change my data within Mongo DB:

db.TRACKING_DATA.find( { 'eventdata' : { $type : 2 } } ).forEach( 
function (x) { x.eventdata = JSON.parse(x.eventdata); 
    db.TRACKING_DATA.save(x);});

db.TRACKING_DATA.find().forEach(
    function(doc) {
    var QueryDate = doc.eventdata.QueryDate;
        if(QueryDate!=null)
        db.TRACKING_DATA.update({ 
            "_id": doc._id 
                }, { 
                    "$set": {
                            "eventdata.QueryDate": new Date(doc.eventdata.QueryDate)        
                        }
                }    
    ) 
});

db.TRACKING_DATA.find().forEach(
    function(doc) {    
        var Viewed = doc.eventdata.Viewed;
        if (Viewed instanceof Array && Viewed[0]) 
            db.TRACKING_DATA.update({ 
            "_id": doc._id ,"eventdata.Viewed.viewDate":doc.eventdata.Viewed[0].viewDate 
            },{ 
                "$set": {              
                        "eventdata.Viewed.$.viewDate": new Date( doc.eventdata.Viewed[0].viewDate)        
                        }
                }  
            ) 
        });

db.TRACKING_DATA.find().forEach(
    function(doc) {
        var Viewed = doc.eventdata.Viewed;
        if (Viewed instanceof Array && Viewed[0]) {
            var Message = Viewed[0].Message;
            if (Message instanceof Array && Message[0]) {

                db.TRACKING_DATA.update({
                    "_id" : doc._id,
                    "eventdata.Viewed.viewDate" : Viewed[0].viewDate
                }, {
                    "$set" : {
                        "eventdata.Viewed.$.Message.0.MessageDate" : new Date(Message[0].MessageDate)
                    }
                });
            }
        }
    });

 db.TRACKING_DATA.find().forEach(     
    function(doc) {         
        var Viewed = doc.eventdata.Viewed;         
            if (Viewed instanceof Array && Viewed[0]) {             
                var Message = Viewed[0].Message;             
                    if (Message instanceof Array && Message[0]) {                  
                        db.TRACKING_DATA.update({                     
                            "_id" : doc._id,                     
                                "eventdata.Viewed.viewDate" : Viewed[0].viewDate                 
                                }, {                     
                                "$set" : {                         
                                    "eventdata.Viewed.$.Message.0.Response.0.ResponseDate" : new Date(Message[0].Response[0].ResponseDate)                     
                                    }                 
                                });             
                            }         
                        }     
                    });

It runs, however, it is taking a very long time ~4hrs to change ~8million documents. How can I speed up the efficiency of this, to reduce the time it takes to run?

Here is the MongoDB structure.

{
_id: "fe50fdee-4ea3-4824-94af-f369633c0c7a",
_class: "com.tracking.daoservice.model.TrackingData",
modified: ISODate("2014-09-10T23:38:48.407Z"),
eventtype: "William-Test",
eventdata: {
    QueryDate: "01-APR-2014",
    SearchQuery: {
        keyword: "Java",
        location: "Santa Clara, CA",
        Facet: "skill~java~perl|workAuth~USC",
        SearchAgentId: "4299"
    },
    Viewed: [
        {
            ViewedID: "8992ade400a",
            Dockey: "3323aba3233",
            PID: "32399a",
            actionsTaken: "email|direct message|report seeker",
            viewDate: "01-APR-2014",
            MessageSent: "true",
            Message: [
                {
                    MessageID: "123aca323",
                    Delivered: "True",
                    Opened: "True",
                    ClickThroughRate: "NotBad",
                    MessageDate: "02-APR-2014",
                    Response: [
                        {
                            ResponseId: "a323a9da",
                            ResponseDate: "23-APR-2014"
                        }
                    ]
                }
            ]
        }
    ]
},
eventsource: "API-Dev Test - JMachine",
sourceip: "myIp",
entityid: "TmoneyBunnyWunny",
groupid: "Dice",
datecreated: ISODate("2014-09-10T23:38:48.405Z")
}

Welcome to Code Review! To make life easier for reviewers, please add sufficient context to your question. The more you tell us about what your code does and what the purpose of doing that is, the easier it will be for reviewers to help you. See also this meta question. In this case, we don't really have a clue about what data your database stores. — Simon Forsberg
– Simon Forsberg, Commented Oct 2, 2014 at 22:24

konijn · Accepted Answer · 2014-10-03 13:49:43Z

Interesting question,

this should go much faster if you

Have 1 find().forEach( only
Do your 4 if statements inside that loop and update an object with the required values
Do one single update per found record at the end with the object you were updating

Something this like ought to work ( again, assuming that criteria only requires _id )

db.TRACKING_DATA.find( { 'eventdata' : { $type : 2 } } ).forEach( 
function (x) { x.eventdata = JSON.parse(x.eventdata); 
    db.TRACKING_DATA.save(x);});

db.TRACKING_DATA.find().forEach( function(doc) {

  var searchCriteria = { "_id": doc._id  },
      set = {};

  //Check 1
  var QueryDate = doc.eventdata.QueryDate; 
  if( (QueryDate!=null) ){
    set["eventdata.QueryDate"] = new Date(doc.eventdata.QueryDate);
  }  

  //Check 2
  var Viewed = doc.eventdata.Viewed;
  if (Viewed instanceof Array && Viewed[0]){
    set["eventdata.Viewed.$.viewDate"] = new Date( doc.eventdata.Viewed[0].viewDate);       


    //Check 3
    var Message = Viewed[0].Message; 
    if (Message instanceof Array && Message[0]){     
      set["eventdata.Viewed.$.Message.0.MessageDate"] = new Date(Message[0].MessageDate);

      //Check 4
      var Response = Message[0].Response;       

      if( Response instanceof Array & Response[0]){
        set["eventdata.Viewed.$.Message.0.Response.0.ResponseDate"] = new Date(Message[0].Response[0].ResponseDate);
      }
    }
  }
  //Do we have anything to set ?
  if( Object.keys(set).length ){
    db.TRACKING_DATA.update( searchCriteria, { "$set" : set } );
  }
});

Note that this refactoring exercise brought out a possible flaw in your code. In check 4 in your code you are not checking for the Reponse object, you simply copy pasted the code from the 3rd check which only checks up to Message.

Hi have added a data JSON to give some insight into the structure of the data I am working on — Will
– Will, Commented Oct 2, 2014 at 22:43
In response to item 3. I am not sure that it is possible given that inner doc of event data is free flowing meaning that it will not always contain all the inner values such as QueryData etc, which is why I split these up. — Will
– Will, Commented Oct 3, 2014 at 1:43
Still looking at this, why do your search queries have criteria other than _id if _id is supposed to be unique? stackoverflow.com/questions/9932631/… — konijn
– konijn, Commented Oct 3, 2014 at 13:29

Stack Exchange Network

Migrating Data in Mongo

1 Answer 1

You must log in to answer this question.

Hot Network Questions

Migrating Data in Mongo

1 Answer 1

You must log in to answer this question.

Related

Hot Network Questions