How can I make my mongodb faster on just simple inserts? I am getting less than 3k insert per sec. I started with an empty data directory. Below please find my scripts and stats.
$ cat startMongoSingle.sh
#!/bin/sh
set -o xtrace
set -o nounset
set -o errexit
./killAllMongo.sh
sleepTime=3
sleep $sleepTime
export dataDir=c:/data
rm -Rf $dataDir
mkdir -p $dataDir
pushd $dataDir
mkdir -p log
mkdir -p configdb0 configdb1 configdb2
mkdir -p shard0 shard1 shard2 shard3
mongod --profile=1 --slowms=15 --dbpath $dataDir/shard0 --logpath $dataDir/log/shard0.log
==============================
+ set -o nounset
+ set -o errexit
+ mongo statAgg.js
MongoDB shell version: 2.8.0-rc4
connecting to: test
+ echo 'db.currentOp()'
+ mongo
MongoDB shell version: 2.8.0-rc4
connecting to: test
{ "inprog" : [ ] }
bye
+ echo 'db.serverStatus()'
+ mongo
MongoDB shell version: 2.8.0-rc4
connecting to: test
{
"host" : "yani5",
"version" : "2.8.0-rc4",
"process" : "C:\\opt\\mongodb\\bin\\
"pid" : NumberLong(1548),
"uptime" : 447,
"uptimeMillis" : NumberLong(446567),
"uptimeEstimate" : 445,
"localTime" : ISODate("2015-01-03T02:24:44.
"asserts" : {
"regular" : 0,
"warning" : 0,
"msg" : 0,
"user" : 0,
"rollovers" : 0
},
"backgroundFlushing" : {
"flushes" : 7,
"total_ms" : 1033,
"average_ms" : 147.57142857142858,
"last_ms" : 512,
"last_finished" : ISODate("2015-01-03T02:24:18.
},
"connections" : {
"current" : 14,
"available" : 999986,
"totalCreated" : NumberLong(41)
},
"cursors" : {
"note" : "deprecated, use server status metrics",
"clientCursors_size" : 0,
"totalOpen" : 0,
"pinned" : 0,
"totalNoTimeout" : 0,
"timedOut" : 0
},
"dur" : {
"commits" : 30,
"journaledMB" : 0.28672,
"writeToDataFilesMB" : 0.710215,
"compression" : 0.3768535775591265,
"commitsInWriteLock" : 0,
"earlyCommits" : 0,
"timeMs" : {
"dt" : 3064,
"prepLogBuffer" : 0,
"writeToJournal" : 14,
"writeToDataFiles" : 1,
"remapPrivateView" : 1
}
},
"extra_info" : {
"note" : "fields vary by platform",
"page_faults" : 184272,
"usagePageFileMB" : 115,
"totalPageFileMB" : 81785,
"availPageFileMB" : 73705,
"ramMB" : 32715
},
"globalLock" : {
"totalTime" : NumberLong(446566000),
"currentQueue" : {
"total" : 0,
"readers" : 0,
"writers" : 0
},
"activeClients" : {
"total" : 20,
"readers" : 0,
"writers" : 0
}
},
"network" : {
"bytesIn" : 102773881,
"bytesOut" : 11461953,
"numRequests" : 191528
},
"opcounters" : {
"insert" : 191246,
"query" : 9,
"update" : 0,
"delete" : 2,
"getmore" : 0,
"command" : 268
},
"opcountersRepl" : {
"insert" : 0,
"query" : 0,
"update" : 0,
"delete" : 0,
"getmore" : 0,
"command" : 0
},
"storageEngine" : {
"name" : "mmapv1"
},
"writeBacksQueued" : false,
"mem" : {
"bits" : 64,
"resident" : 145,
"virtual" : 723,
"supported" : true,
"mapped" : 288,
"mappedWithJournal" : 576
},
"metrics" : {
"commands" : {
"aggregate" : {
"failed" : NumberLong(0),
"total" : NumberLong(5)
},
"buildInfo" : {
"failed" : NumberLong(0),
"total" : NumberLong(9)
},
"collStats" : {
"failed" : NumberLong(0),
"total" : NumberLong(3)
},
"delete" : {
"failed" : NumberLong(0),
"total" : NumberLong(2)
},
"getnonce" : {
"failed" : NumberLong(0),
"total" : NumberLong(4)
},
"insert" : {
"failed" : NumberLong(0),
"total" : NumberLong(191246)
},
"isMaster" : {
"failed" : NumberLong(0),
"total" : NumberLong(43)
},
"listDatabases" : {
"failed" : NumberLong(0),
"total" : NumberLong(4)
},
"ping" : {
"failed" : NumberLong(0),
"total" : NumberLong(80)
},
"replSetGetStatus" : {
"failed" : NumberLong(18),
"total" : NumberLong(18)
},
"serverStatus" : {
"failed" : NumberLong(0),
"total" : NumberLong(75)
},
"whatsmyuri" : {
"failed" : NumberLong(0),
"total" : NumberLong(27)
}
},
"cursor" : {
"timedOut" : NumberLong(0),
"open" : {
"noTimeout" : NumberLong(0),
"pinned" : NumberLong(0),
"total" : NumberLong(0)
}
},
"document" : {
"deleted" : NumberLong(171767),
"inserted" : NumberLong(191246),
"returned" : NumberLong(8),
"updated" : NumberLong(0)
},
"getLastError" : {
"wtime" : {
"num" : 0,
"totalMillis" : 0
},
"wtimeouts" : NumberLong(0)
},
"operation" : {
"fastmod" : NumberLong(0),
"idhack" : NumberLong(0),
"scanAndOrder" : NumberLong(0)
},
"queryExecutor" : {
"scanned" : NumberLong(0),
"scannedObjects" : NumberLong(9)
},
"record" : {
"moves" : NumberLong(0)
},
"repl" : {
"apply" : {
"batches" : {
"num" : 0,
"totalMillis" : 0
},
"ops" : NumberLong(0)
},
"buffer" : {
"count" : NumberLong(0),
"maxSizeBytes" : 268435456,
"sizeBytes" : NumberLong(0)
},
"network" : {
"bytes" : NumberLong(0),
"getmores" : {
"num" : 0,
"totalMillis" : 0
},
"ops" : NumberLong(0),
"readersCreated" : NumberLong(0)
},
"preload" : {
"docs" : {
"num" : 0,
"totalMillis" : 0
},
"indexes" : {
"num" : 0,
"totalMillis" : 0
}
}
},
"storage" : {
"freelist" : {
"search" : {
"bucketExhausted" : NumberLong(0),
"requests" : NumberLong(192032),
"scanned" : NumberLong(0)
}
}
},
"ttl" : {
"deletedDocuments" : NumberLong(0),
"passes" : NumberLong(7)
}
},
"ok" : 1
}
bye
+ echo 'db.serverBuildInfo()'
+ mongo
MongoDB shell version: 2.8.0-rc4
connecting to: test
{
"version" : "2.8.0-rc4",
"gitVersion" : "
"targetMinOS" : "Windows 7/Windows Server 2008 R2",
"OpenSSLVersion" : "",
"sysInfo" : "windows sys.getwindowsversion(major=6, minor=1, build=7601, platform=2, service_pack='Service Pack 1') BOOST_LIB_VERSION=1_49",
"loaderFlags" : "/nologo /DEBUG /INCREMENTAL:NO /LARGEADDRESSAWARE /OPT:REF",
"compilerFlags" : "/TP /nologo /EHsc /W3 /wd4355 /wd4800 /wd4267 /wd4244 /wd4290 /wd4068 /wd4351 /we4099 /Z7 /errorReport:none /MT /O2 /Oy- /Gw /Gy /Zc:inline",
"allocator" : "system",
"versionArray" : [
2,
8,
0,
-6
],
"javascriptEngine" : "V8",
"bits" : 64,
"debug" : false,
"maxBsonObjectSize" : 16777216,
"ok" : 1
}
bye
+ echo 'db['\''system.profile'\''].
+ mongo
MongoDB shell version: 2.8.0-rc4
connecting to: test
{ "op" : "insert", "ns" : "test.ticks", "query" : { "_id" : ObjectId("
{ "op" : "insert", "ns" : "test.ticks", "query" : { "_id" : ObjectId("
{ "op" : "insert", "ns" : "test.ticks", "query" : { "_id" : ObjectId("
{ "op" : "insert", "ns" : "test.ticks", "query" : { "_id" : ObjectId("
{ "op" : "insert", "ns" : "test.ticks", "query" : { "_id" : ObjectId("
{ "op" : "insert", "ns" : "test.ticks", "query" : { "_id" : ObjectId("
{ "op" : "insert", "ns" : "test.ticks", "query" : { "_id" : ObjectId("
{ "op" : "remove", "ns" : "test.ticks", "query" : { }, "ndeleted" : 171767, "keyUpdates" : 0, "numYield" : 1341, "millis" : 1205, "execStats" : { }, "ts" : ISODate("2015-01-03T02:23:59.
{ "op" : "insert", "ns" : "test.ticks", "query" : { "_id" : ObjectId("
{ "op" : "insert", "ns" : "test.ticks", "query" : { "_id" : ObjectId("
{ "op" : "insert", "ns" : "test.ticks", "query" : { "_id" : ObjectId("
bye
+ mongostat --all -n 20
insert query update delete getmore command flushes mapped vsize res non-mapped faults idx miss % qr|qw ar|aw netIn netOut conn time
435 *0 *0 *0 0 1|0 0 288.0m 723.0m 145.0m 435.0m 183 0 0|0 0|0 233k 34k 14 10:24:45
439 *0 *0 *0 0 1|0 0 288.0m 723.0m 145.0m 435.0m 187 0 0|0 0|0 235k 34k 14 10:24:46
441 *0 *0 *0 0 1|0 0 288.0m 723.0m 145.0m 435.0m 184 0 0|0 0|0 236k 34k 14 10:24:47
439 *0 *0 *0 0 1|0 0 288.0m 723.0m 145.0m 435.0m 187 0 0|0 0|0 235k 34k 14 10:24:48
445 *0 *0 *0 0 2|0 0 288.0m 723.0m 145.0m 435.0m 183 0 0|0 0|0 239k 34k 14 10:24:49
442 *0 *0 *0 0 3|0 0 288.0m 723.0m 145.0m 435.0m 187 0 0|0 0|0 237k 34k 14 10:24:50
442 *0 *0 *0 0 1|0 0 288.0m 723.0m 145.0m 435.0m 187 0 0|0 0|0 237k 34k 14 10:24:51
439 *0 *0 *0 0 1|0 0 288.0m 723.0m 145.0m 435.0m 182 0 0|0 0|0 235k 34k 14 10:24:52
441 *0 *0 *0 0 1|0 0 288.0m 723.0m 145.0m 435.0m 183 0 0|0 0|0 236k 34k 14 10:24:53
435 *0 *0 *0 0 2|0 0 288.0m 723.0m 145.0m 435.0m 141 0 0|0 0|0 233k 34k 14 10:24:54
insert query update delete getmore command flushes mapped vsize res non-mapped faults idx miss % qr|qw ar|aw netIn netOut conn time
435 *0 *0 *0 0 1|0 0 288.0m 723.0m 145.0m 435.0m 187 0 0|0 0|0 233k 34k 14 10:24:55
436 *0 *0 *0 0 1|0 0 288.0m 723.0m 145.0m 435.0m 186 0 0|0 0|0 234k 34k 14 10:24:56
441 *0 *0 *0 0 1|0 0 288.0m 723.0m 145.0m 435.0m 183 0 0|0 0|0 236k 34k 14 10:24:57
442 *0 *0 *0 0 1|0 0 288.0m 723.0m 145.0m 435.0m 187 0 0|0 0|0 237k 34k 14 10:24:58
441 *0 *0 *0 0 2|0 0 288.0m 723.0m 145.0m 435.0m 187 0 0|0 0|0 236k 34k 14 10:24:59
445 *0 *0 *0 0 1|0 0 288.0m 723.0m 145.0m 435.0m 186 0 0|0 0|0 239k 34k 14 10:25:00
444 *0 *0 *0 0 1|0 0 288.0m 723.0m 145.0m 435.0m 184 0 0|0 0|0 238k 34k 14 10:25:01
444 *0 *0 *0 0 3|0 0 288.0m 723.0m 145.0m 435.0m 186 0 0|0 0|0 238k 34k 14 10:25:02
442 *0 *0 *0 0 1|0 0 288.0m 723.0m 145.0m 435.0m 187 0 0|0 0|0 237k 34k 14 10:25:03
445 *0 *0 *0 0 2|0 0 288.0m 723.0m 145.0m 435.0m 187 0 0|0 0|0 239k 34k 14 10:25:04
I have tried on both windows and linux. Both version 2.6.6 and 2.8. The speeds are similar. Below please find the C++ program that generated the insert statements.
// mongo_cxx_load_gen.cpp
#include <cstdlib>
#include <iostream>
#include <fstream>
#include <vector>
#include <random>
#include <thread>
#include <boost/format.hpp>
#ifdef _WIN32
#include <WinSock2.h>
#endif
#include "mongo/client/dbclient.h" // for the driver
using namespace std;
using mongo::be;
using mongo::bo;
using mongo::bob;
vector<string> loadTextFile(string filename) {
ifstream ifs(filename);
string line;
vector<string> result;
while (getline(ifs, line)) {
result.push_back(line);
}
return result;
}
vector<string> symbols = loadTextFile("symbols.txt");
vector<string> feeds = loadTextFile("feeds.txt");
int threadId = std::this_thread::get_id().
std::random_device rd;
std::mt19937 gen(rd());
std::geometric_distribution<> geodist;
bo getBsonFromJson(string line) {
string json = string("{")
.append("\"lineno\": ").append('"' + line + '"')
.append(", \"time\": ").append("Timestamp(
.append(", \"symbol\": ").append('"' + symbols[geodist(gen) % symbols.size()] + '"')
.append(", \"feed\": ").append('"' + feeds[geodist(gen) % feeds.size()] + '"')
;
for (int c = 0; c < 20; ++c) {
string fieldname = str(boost::format("price%02d: ") % c);
json.append(", ").append(fieldname).append(
}
json.append("}");
return mongo::fromjson(json);
}
bo getBsonFromBuilder(string line) {
const mongo::OID generated = mongo::OID::gen();
bob b;
b.append("_id", generated);
b.append("lineno", line);
b.appendTimestamp("time");
b.append("symbol", symbols[geodist(gen) % symbols.size()]);
b.append("feed", feeds[geodist(gen) % feeds.size()]);
for (int c = 0; c < 20; ++c) {
string fieldname = str(boost::format("price%02d") % c);
b.appendNumber(fieldname, (double)geodist(gen));
}
bo result = b.obj();
return result;
}
void run(string hostColonPort) {
mongo::DBClientConnection c;
c.connect(hostColonPort);
string line;
for (int a = 0; getline(cin, line); ++a)
{
//bo result = getBsonFromJson(line);
bo result = getBsonFromBuilder(line);
cout << hostColonPort + " - " << result << endl;
c.insert("test.ticks", result);
}
}
int main(int, char** argv) {
mongo::client::initialize();
try {
run(string(argv[1]));
std::cout << "connected ok" << std::endl;
}
catch (const mongo::DBException &e) {
std::cout << "caught " << e.what() << std::endl;
}
return EXIT_SUCCESS;
}
btw, my target is scaling out/up to 100k+ insert / sec
Is there a single lock being contended making everything serial?
댓글 없음:
댓글 쓰기