Mongo Map / Substitution Reduction in Large Collections

We have a seemingly simple work with a card / reduction, which processes registration data daily. On the development server, we can run this work on a very large number of documents, ~ 1M, and this takes about a minute without problems. We transfer the work to production servers, which are Amazon EC2 servers, the work will scroll about 50% of the lines at a very high speed, and then scan the rest of the data. It may take several hours to go through several hundred thousand documents, instead of the expected minute or two. Therefore, I hope that we made an obvious mistake in working with the map / reduction.

Here is an example input document:

{
    "_id": ObjectId ("4f147a92d72b292c02000057"),
    "cid": 25,
    "ip": "123.45.67.89",
    "b": "Mozilla / 5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit / 535.7 (KHTML, like Gecko) Chrome / 16.0.912.63 Safari / 535.7",
    "r": "",
    "l": "en-US, en; q = 0.8",
    "ts": ISODate ("2012-01-16T19: 29: 22Z"),
    "s": 0,
    "cv": "4f143a5fd72b292d7f000007",
    "c": ""
}

We request only the range _id.

Here is the card code:

function () { 
    var browser = {}
    , referrer = {};
    browser [this.b] = {
        'count': 1
    };
    referrer [this.r] = {
        'count': 1
    };
    var objEmit = {
        'count': 1
        , 'browsers': browser
        , 'referrers': referrer
    };
    var date = this._id.getTimestamp ();
    date.setHours (0);
    date.setMinutes (0);
    date.setSeconds(0);
    emit({'cv' : this.cv, 'date' : date, 'cid' : this.cid }, objEmit);
};

:

function (key, emits) {
    var total = 0
    ,browsers = {}
    ,referrers = {};
    for (var i in emits) {
        total += emits[i].count;
        for (var key in emits[i].browsers) {
            if (emits[i].browsers.hasOwnProperty(key)) {
                !(browsers[key]) && (browsers[key] = { count : 0 });
                browsers[key].count +=  emits[i].browsers[key].count;
            }
        }
        for (var key in emits[i].referrers) {
            if (emits[i].referrers.hasOwnProperty(key)) {
                !(referrers[key]) && (referrers[key] = { count : 0 });
                referrers[key].count += emits[i].referrers[key].count;
            }
        }
    }
    return {'count' : total, 'browsers' : browsers, 'referrers' : referrers}
};

, map/reduce "merge", true.

.

+3
1

, dev production, dev , , , , ?

, Micro? , , , , (- , , ​​Linux ).

Micro Small, , , "" ( ), MongoDB, , .

, , , .

0

All Articles