搭建Mongo Cluster(Sharding + replica set)

搭建Mongo Cluster( Sharding + replica set )
===========================================================
原理: sharding是可以凌驾也replica set之上的, 由下面的命令支持
sh.addShard ( /,,)
比如先配置10个replica sets, 每个replica set有3个成员，就可以组成30台的cluster

同时使用keyfile和用户认证方式，有人说这样性能慢10倍，有待验证

0. 准备
rm -fr /etc/mongodb
rm -fr /data/
mkdir -p /data/
mkdir -p /etc/mongodb

echo "10.2.9.32 dbsvr01" >>/etc/hosts
echo "10.2.10.96 dbsvr03" >>/etc/hosts
echo "10.2.10.17 dbsvr02" >>/etc/hosts

wget
tar -zxvf mongodb-linux-x86_64-2.2.1.tgz
mv mongodb-linux-x86_64-2.2.1/bin/* /usr/bin/

1. 在3台物理机器上建立sh01
echo "SKey4XCluster erjfouyhr 7yh6748dj uy4693ujg" > /etc/mongodb/cluster01.key
chmod 600 /etc/mongodb/cluster01.key
mkdir -p /data/sh01/tmp

tee /etc/mongodb/sh01.cnf
dbpath=/data/sh01
logpath=/data/sh01/sh01.log
repairpath=/data/sh01/tmp
fork=1
auth=true
port=2001
journal=1
directoryperdb=1
rest=1
shardsvr=true
keyFile=/etc/mongodb/cluster01.key
replSet=rs01

tee /etc/mongodb/start_sh01.sh
mongod -f /etc/mongodb/sh01.cnf
chmod ug+x /etc/mongodb/start_sh01.sh

启动3个实例
2 初始化rs01
mongo admin --port 2001
rs.initiate({_id: 'rs01', members: [
{_id: 0, host: 'dbsvr01:2001', arbiterOnly: true},
{_id: 1, host: 'dbsvr02:2001'},
{_id: 2, host: 'dbsvr03:2001'}]
});
rs.status();
db.addUser('admin','admin123');
用户添加后，马上生效，需要重新登录才能使用命令

执行命令的机器必须首先成为primary

3. 在3台物理机器上建立sh02
echo "SKey4XCluster erjfouyhr 7yh6748dj uy4693ujg" > /etc/mongodb/cluster01.key
chmod 600 /etc/mongodb/cluster01.key

mkdir -p /data/sh02/tmp

tee /etc/mongodb/sh02.cnf
dbpath=/data/sh02
logpath=/data/sh02/sh02.log
repairpath=/data/sh02/tmp
fork=1
auth=true
port=2002
journal=1
directoryperdb=1
rest=1
shardsvr=true
keyFile=/etc/mongodb/cluster01.key
replSet=rs02

tee /etc/mongodb/start_sh02.sh
mongod -f /etc/mongodb/sh02.cnf
chmod ug+x /etc/mongodb/start_sh02.sh
/etc/mongodb/start_sh02.sh

启动3个实例
4 初始化rs02
mongo admin --port 2002
rs.initiate({_id: 'rs02', members: [
{_id: 0, host: 'dbsvr01:2002'},
{_id: 1, host: 'dbsvr02:2002', arbiterOnly: true},
{_id: 2, host: 'dbsvr03:2002'}]
});
rs.status();
db.addUser('admin','admin123');


5. 在3台物理机器上建立sh03
echo "SKey4XCluster erjfouyhr 7yh6748dj uy4693ujg" > /etc/mongodb/cluster01.key
chmod 600 /etc/mongodb/cluster01.key

mkdir -p /data/sh03/tmp


tee /etc/mongodb/sh03.cnf
dbpath=/data/sh03
logpath=/data/sh03/sh03.log
repairpath=/data/sh03/tmp
fork=1
auth=true
port=2003
journal=1
directoryperdb=1
rest=1
shardsvr=true
keyFile=/etc/mongodb/cluster01.key
replSet=rs03

tee /etc/mongodb/start_sh03.sh
mongod -f /etc/mongodb/sh03.cnf
chmod ug+x /etc/mongodb/start_sh03.sh
/etc/mongodb/start_sh03.sh
mongo admin --port 2003

启动3个实例
6 初始化rs03
mongo admin --port 2003
rs.initiate({_id: 'rs03', members: [
{_id: 0, host: 'dbsvr01:2003'},
{_id: 1, host: 'dbsvr02:2003'},
{_id: 2, host: 'dbsvr03:2003', arbiterOnly: true}]
});
rs.status();
db.addUser('admin','admin123');

7. 检查复制
rs.status();
Primry的最先OK, 其他的UNKNOWN->Startup2-->"RECOVERING-->SECONDARY
跑rs.initiate命令的server会首先成为Primary

8 在3台服务器上配置config server
echo "SKey4XCluster erjfouyhr 7yh6748dj uy4693ujg" > /etc/mongodb/cluster01.key
chmod 600 /etc/mongodb/cluster01.key

mkdir -p /data/config/tmp

tee /etc/mongodb/config.cnf
dbpath=/data/config
logpath=/data/config/shard.log
repairpath=/data/config/tmp
fork=1
port=2101
journal=1
directoryperdb=1
rest=1
configsvr=true
keyFile=/etc/mongodb/cluster01.key

echo "mongod -f /etc/mongodb/config.cnf" > /etc/mongodb/start_config.sh
chmod ug+x /etc/mongodb/start_config.sh
/etc/mongodb/start_config.sh

mongo admin --port 2101
db.addUser('admin','admin123');


10. 在3台服务器上配置Route Server
echo "SKey4XCluster erjfouyhr 7yh6748dj uy4693ujg" > /etc/mongodb/cluster01.key
chmod 600 /etc/mongodb/cluster01.key

mkdir -p /data/route/tmp

tee /etc/mongodb/route.cnf
logpath=/data/route/route.log
fork=1
port=2102
chunkSize=1
configdb=dbsvr02:2101,dbsvr01:2101,dbsvr03:2101
keyFile=/etc/mongodb/cluster01.key


echo "mongos -f /etc/mongodb/route.cnf" > /etc/mongodb/start_route.sh
chmod ug+x /etc/mongodb/start_route.sh
/etc/mongodb/start_route.sh

11 初始化sharding
mongo admin --port 2102
sh.addShard( "rs01/dbsvr02:2001" );
sh.addShard( "rs02/dbsvr01:2002" );
sh.addShard( "rs03/dbsvr03:2003" );
db.runCommand({ listshards: 1 });

现在addShard可以自动扫描其成员，而不需要指定每个成员了

12 验证状态
mongos> db.runCommand({ listshards: 1 });
{
"shards" : [
{
"_id" : "rs01",
"host" : "rs01/dbsvr02:2001,dbsvr03:2001"
},
{
"_id" : "rs02",
"host" : "rs02/dbsvr01:2002,dbsvr03:2002"
},
{
"_id" : "rs03",
"host" : "rs03/dbsvr01:2003,dbsvr02:2003"
}
],
"ok" : 1
}
mongos> sh.status();
--- Sharding Status ---
sharding version: { "_id" : 1, "version" : 3 }
shards:
{ "_id" : "rs01", "host" : "rs01/dbsvr02:2001,dbsvr03:2001" }
{ "_id" : "rs02", "host" : "rs02/dbsvr01:2002,dbsvr03:2002" }
{ "_id" : "rs03", "host" : "rs03/dbsvr01:2003,dbsvr02:2003" }
databases:
{ "_id" : "admin", "partitioned" : false, "primary" : "config" }

13 开启sharding
mongo -u admin -p admin123 admin --port 2102
sh.enableSharding( "demo" );
sh.shardCollection("demo.hwz",{_id:1});

mongos> sh.status();
--- Sharding Status ---
sharding version: { "_id" : 1, "version" : 3 }
shards:
{ "_id" : "rs01", "host" : "rs01/dbsvr02:2001,dbsvr03:2001" }
{ "_id" : "rs02", "host" : "rs02/dbsvr01:2002,dbsvr03:2002" }
{ "_id" : "rs03", "host" : "rs03/dbsvr01:2003,dbsvr02:2003" }
databases:
{ "_id" : "admin", "partitioned" : false, "primary" : "config" }
{ "_id" : "demo", "partitioned" : true, "primary" : "rs02" }
demo.hwz chunks:
rs02 1
{ "_id" : { $minKey : 1 } } -->> { "_id" : { $maxKey : 1 } } on : rs02 Timestamp(1000, 0)
use demo;
db.addUser('demo','demo456');

14. 测试插入数据
mongo -u demo -p demo456 dbsvr03:2102/demo
for (var i=1;i<=100000;i++) db.hwz.insert( {id:i,name:'jackaaa',addr:'shanghai'} );
mongos> sh.status();
--- Sharding Status ---
sharding version: { "_id" : 1, "version" : 3 }
shards:
{ "_id" : "rs01", "host" : "rs01/dbsvr02:2001,dbsvr03:2001" }
{ "_id" : "rs02", "host" : "rs02/dbsvr01:2002,dbsvr03:2002" }
{ "_id" : "rs03", "host" : "rs03/dbsvr01:2003,dbsvr02:2003" }
databases:
{ "_id" : "admin", "partitioned" : false, "primary" : "config" }
{ "_id" : "demo", "partitioned" : true, "primary" : "rs02" }
demo.hwz chunks:
rs02 2
rs03 2
rs01 3
{ "_id" : { $minKey : 1 } } -->> { "_id" : ObjectId("50b6f9a771f436be35c037ae") } on : rs02 Timestamp(2000, 1)
{ "_id" : ObjectId("50b6f9a771f436be35c037ae") } -->> { "_id" : ObjectId("50b6f9a971f436be35c056ea") } on : rs02 Timestamp(1000, 3)
{ "_id" : ObjectId("50b6f9a971f436be35c056ea") } -->> { "_id" : ObjectId("50b6f9ad71f436be35c09294") } on : rs03 Timestamp(3000, 1)
{ "_id" : ObjectId("50b6f9ad71f436be35c09294") } -->> { "_id" : ObjectId("50b6f9b271f436be35c0d381") } on : rs03 Timestamp(4000, 0)
{ "_id" : ObjectId("50b6f9b271f436be35c0d381") } -->> { "_id" : ObjectId("50b6f9b571f436be35c10d98") } on : rs01 Timestamp(4000, 1)
{ "_id" : ObjectId("50b6f9b571f436be35c10d98") } -->> { "_id" : ObjectId("50b6f9bc71f436be35c1a4a3") } on : rs01 Timestamp(4000, 2)
{ "_id" : ObjectId("50b6f9bc71f436be35c1a4a3") } -->> { "_id" : { $maxKey : 1 } } on : rs01 Timestamp(4000, 3)
mongos>

自动sharding的算法很不均匀，插入是3:3:10经过自动Balance后才变为5:5:6, rs01的数据明显偏多
rs02 3
rs03 3
rs01 10

rs02 5
rs03 5
rs01 6