redis集群配置

 

1.    部署

环境：

OS:Centos 7

Redis: 3.2.9

 

3主3从配置

序号	主	从
1	127.0.0.1:7000	127.0.0.1:7003
2	127.0.0.1:7001	127.0.0.1:7004
3	127.0.0.1:7002	127.0.0.1:7005

 

1.1     下载redis

直接到官网下载，我这里下载的是redis-3.2.9.tar.gz.
下载地址:

1.2     安装软件

1.2.1    解压缩、安装

[root@hxl soft]#tar –zxvf redis-3.2.9.tar.gz

[root@hxl soft]# cp -R ./redis-3.2.9 /opt/

[root@izwz9gxsbwybwg9n5xdi47z opt]# cd redis-3.2.9

[root@izwz9gxsbwybwg9n5xdi47z redis-3.2.9]# make

[root@izwz9gxsbwybwg9n5xdi47z redis-3.2.9]# make test

cd src && make test

make[1]: Entering directory `/opt/redis-3.2.9/src'

You need tcl 8.5 or newer in order to run the Redis test

make[1]: *** [test] Error 1

make[1]: Leaving directory `/opt/redis-3.2.9/src'

make: *** [test] Error 2

需要安装tcl

[root@izwz9gxsbwybwg9n5xdi47z soft]# yum -y install tcl

[root@izwz9gxsbwybwg9n5xdi47z redis-3.2.9]# cd src && make install

1.2.2    创建集群目录

[root@izwz9gxsbwybwg9n5xdi47z opt]#cd /opt && mkdir redis-cluster && cd redis-cluster

[root@izwz9gxsbwybwg9n5xdi47z cluster]# mkdir 7000 7001 7002 7003 7004 7005

1.2.3    配置文件

拷贝redis.conf文件到7000 7001 7002 7003 7004 7005 6个子目录

[root@izwz9gxsbwybwg9n5xdi47z redis-3.2.9]# pwd

/opt/redis-3.2.9

[root@izwz9gxsbwybwg9n5xdi47z redis-3.2.9]# cp redis.conf /opt/cluster/7000/

[root@izwz9gxsbwybwg9n5xdi47z redis-3.2.9]# cp redis.conf /opt/cluster/7001/

[root@izwz9gxsbwybwg9n5xdi47z redis-3.2.9]# cp redis.conf /opt/cluster/7002/

[root@izwz9gxsbwybwg9n5xdi47z redis-3.2.9]# cp redis.conf /opt/cluster/7003/

[root@izwz9gxsbwybwg9n5xdi47z redis-3.2.9]# cp redis.conf /opt/cluster/7004/

[root@izwz9gxsbwybwg9n5xdi47z redis-3.2.9]# cp redis.conf /opt/cluster/7005/

修改配置文件以7000为例子(redis.cnf)

提前创建好配置文件中的目录

mkdir -p /opt/redis-cluster/log

mkdir -p /opt/redis-cluster/ nodes

mkdir -p /opt/redis-cluster/ rbd

mkdir -p /opt/redis-cluster/ run


配置文件内如如下:

daemonize yes

pidfile /opt/redis-cluster/redis-7000.pid

port 7000

tcp-backlog 511

timeout 0

tcp-keepalive 0

loglevel notice

logfile "/opt/redis-cluster/log/redis-7000.log"

databases 16

save 900 1

save 300 10

save 60 10000

stop-writes-on-bgsave-error yes

rdbcompression yes

rdbchecksum yes

dbfilename dump-7000.rdb

dir /opt/redis-cluster/rbd

slave-serve-stale-data yes

slave-read-only yes

repl-diskless-sync no

repl-diskless-sync-delay 5

repl-disable-tcp-nodelay no

slave-priority 100

appendonly yes

appendfilename "redis-7000.aof"

appendfsync everysec

no-appendfsync-on-rewrite no

auto-aof-rewrite-percentage 100

auto-aof-rewrite-min-size 64mb

aof-load-truncated yes

lua-time-limit 5000

cluster-enabled yes

cluster-config-file "nodes-7000.conf" ##这里另外的节点需要修改,发现这里不能使用绝对路径,使用了绝对路径会报Unrecoverable error: corrupted cluster config file

cluster-node-timeout 15000

slowlog-log-slower-than 10000

slowlog-max-len 128

latency-monitor-threshold 0

notify-keyspace-events ""

hash-max-ziplist-entries 512

hash-max-ziplist-value 64

list-max-ziplist-entries 512

list-max-ziplist-value 64

set-max-intset-entries 512

zset-max-ziplist-entries 128

zset-max-ziplist-value 64

hll-sparse-max-bytes 3000

activerehashing yes

client-output-buffer-limit normal 0 0 0

client-output-buffer-limit slave 256mb 64mb 60

client-output-buffer-limit pubsub 32mb 8mb 60

hz 10

aof-rewrite-incremental-fsync yes

# Generated by CONFIG REWRITE

#masterauth "richinfo123"

#requirepass "richinfo123"

protected-mode no
bind 192.168.1.118 ##绑定本机的ip地址

[root@izwz9gxsbwybwg9n5xdi47z 7000]# cp redis.conf ../7001/

[root@izwz9gxsbwybwg9n5xdi47z 7000]# cp redis.conf ../7002/

[root@izwz9gxsbwybwg9n5xdi47z 7000]# cp redis.conf ../7003/

[root@izwz9gxsbwybwg9n5xdi47z 7000]# cp redis.conf ../7004/

[root@izwz9gxsbwybwg9n5xdi47z 7000]# cp redis.conf ../7005/

拷贝过去后将红色部分进行修改。

sed -i 's/7000/7001/g' /opt/redis-cluster/7001/redis.conf

sed -i 's/7000/7002/g' /opt/redis-cluster/7002/redis.conf

sed -i 's/7000/7003/g' /opt/redis-cluster/7003/redis.conf

sed -i 's/7000/7004/g' /opt/redis-cluster/7004/redis.conf

sed -i 's/7000/7005/g' /opt/redis-cluster/7005/redis.conf

1.3     启动redis

启动7000

/opt/redis-3.2.9/src/redis-server /opt/redis-cluster/7000/redis.conf

查看日志:

[root@izwz9gxsbwybwg9n5xdi47z log]# more redis-7000.log

984:M 28 May 13:57:15.027 * No cluster configuration found, I'm bc6e5ffeb32d65b9b8d9cdc3c3a1997a2f3b3728

                _._                                                 

           _.-``__ ''-._                                             

      _.-``    `.  `_.  ''-._           Redis 3.2.9 (00000000/0) 64 bit

  .-`` .-```.  ```\/    _.,_ ''-._                                  

 (    '      ,       .-`  | `,    )     Running in cluster mode

 |`-._`-...-` __...-.``-._|'` _.-'|     Port: 7000

 |    `-._   `._    /     _.-'    |     PID: 984

  `-._    `-._  `-./  _.-'    _.-'                                  

 |`-._`-._    `-.__.-'    _.-'_.-'|                                 

 |    `-._`-._        _.-'_.-'    |          

  `-._    `-._`-.__.-'_.-'    _.-'                                  

 |`-._`-._    `-.__.-'    _.-'_.-'|                                 

 |    `-._`-._        _.-'_.-'    |                                 

  `-._    `-._`-.__.-'_.-'    _.-'                                  

      `-._    `-.__.-'    _.-'                                      

          `-._        _.-'                                          

              `-.__.-'                                               

984:M 28 May 13:57:15.031 # WARNING: The TCP backlog setting of 511 cannot be enforced because /proc/sys/net/core/somaxconn is set t

o the lower value of 128.

984:M 28 May 13:57:15.031 # Server started, Redis version 3.2.9

984:M 28 May 13:57:15.031 # WARNING overcommit_memory is set to 0! Background save may fail under low memory condition. To fix this

issue add 'vm.overcommit_memory = 1' to /etc/sysctl.conf and then reboot or run the command 'sysctl vm.overcommit_memory=1' for this

 to take effect.

984:M 28 May 13:57:15.031 # WARNING you have Transparent Huge Pages (THP) support enabled in your kernel. This will create latency a

nd memory usage issues with Redis. To fix this issue run the command 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' as r

oot, and add it to your /etc/rc.local in order to retain the setting after a reboot. Redis must be restarted after THP is disabled.

984:M 28 May 13:57:15.031 * The server is now ready to accept connections on port 7000

以次启动另外的节点:

/opt/redis-3.2.9/src/redis-server /opt/redis-cluster/7001/redis.conf

/opt/redis-3.2.9/src/redis-server /opt/redis-cluster/7002/redis.conf

/opt/redis-3.2.9/src/redis-server /opt/redis-cluster/7003/redis.conf

/opt/redis-3.2.9/src/redis-server /opt/redis-cluster/7004/redis.conf

/opt/redis-3.2.9/src/redis-server /opt/redis-cluster/7005/redis.conf

1.4     创建集群

[root@izwz9gxsbwybwg9n5xdi47z src]# /opt/redis-3.2.9/src/redis-trib.rb create --replicas 1 127.0.0.1:7000 127.0.0.1:7001 127.0.0.1:7002 127.0.0.1:7003 127.0.0.1:7004 127.0.0.1:7005

/usr/bin/env: ruby: No such file or directory

需要安装ruby

[root@izwz9gxsbwybwg9n5xdi47z soft]# yum -y install ruby

/usr/share/rubygems/rubygems/core_ext/kernel_require.rb:55:in `require': cannot load such file -- redis (LoadError)

        from /usr/share/rubygems/rubygems/core_ext/kernel_require.rb:55:in `require'

        from /opt/redis-3.2.9/src/redis-trib.rb:25:in `

'

解决办法:

[root@izwz9gxsbwybwg9n5xdi47z redis-3.2.9]# gem install redis

Fetching: redis-3.3.3.gem (100%)

Successfully installed redis-3.3.3

Parsing documentation for redis-3.3.3

Installing ri documentation for redis-3.3.3

1 gem installed

[root@redis01 ~]# gem install redis
ERROR:  Could not find a valid gem 'redis' (>= 0), here is why:
          Unable to download data from - no such name (latest_specs.4.8.gz)
该错误提示是无法获取到gem安装包，需要手工下载然后安装
[root@redis01 ~]# gem install -l redis-3.3.0.gem

[root@localhost redis_cluster]# gem install redis
Fetching: redis-4.1.2.gem (100%)
ERROR:  Error installing redis:
        redis requires Ruby version >= 2.3.0.
You have mail in /var/spool/mail/root

解决办法参考:
http://blog.chinaunix.net/uid-77311-id-5824669.html


再次执行创建集群命令:

[root@izwz9gxsbwybwg9n5xdi47z redis-3.2.9]# /opt/redis-3.2.9/src/redis-trib.rb create --replicas 1 127.0.0.1:7000 127.0.0.1:7001 127.0.0.1:7002 127.0.0.1:7003 127.0.0.1:7004 127.0.0.1:7005

>>> Creating cluster

>>> Performing hash slots allocation on 6 nodes...

Using 3 masters:

127.0.0.1:7000

127.0.0.1:7001

127.0.0.1:7002

Adding replica 127.0.0.1:7003 to 127.0.0.1:7000

Adding replica 127.0.0.1:7004 to 127.0.0.1:7001

Adding replica 127.0.0.1:7005 to 127.0.0.1:7002

M: bc6e5ffeb32d65b9b8d9cdc3c3a1997a2f3b3728 127.0.0.1:7000

   slots:0-5460 (5461 slots) master

M: 338e7821e3f84ed3124b4e3667d0a49d2e0f602f 127.0.0.1:7001

   slots:5461-10922 (5462 slots) master

M: 410e0ec3d0c88187fc9887232a233e1d26cf5a28 127.0.0.1:7002

   slots:10923-16383 (5461 slots) master

S: 0eca33579c9e9d3c470d591d0f8d709855432db5 127.0.0.1:7003

   replicates bc6e5ffeb32d65b9b8d9cdc3c3a1997a2f3b3728

S: f60895e75326d4d890202ac46f68a5af390ace54 127.0.0.1:7004

   replicates 338e7821e3f84ed3124b4e3667d0a49d2e0f602f

S: b9096b43e1d23d6ac9e79f49a45e8125b22b71be 127.0.0.1:7005

   replicates 410e0ec3d0c88187fc9887232a233e1d26cf5a28

Can I set the above configuration? (type 'yes' to accept)

>>> Nodes configuration updated

>>> Assign a different config epoch to each node

>>> Sending CLUSTER MEET messages to join the cluster

Waiting for the cluster to join...

>>> Performing Cluster Check (using node 127.0.0.1:7000)

M: bc6e5ffeb32d65b9b8d9cdc3c3a1997a2f3b3728 127.0.0.1:7000

   slots:0-5460 (5461 slots) master

   1 additional replica(s)

S: 0eca33579c9e9d3c470d591d0f8d709855432db5 127.0.0.1:7003

   slots: (0 slots) slave

   replicates bc6e5ffeb32d65b9b8d9cdc3c3a1997a2f3b3728

M: 410e0ec3d0c88187fc9887232a233e1d26cf5a28 127.0.0.1:7002

   slots:10923-16383 (5461 slots) master

   1 additional replica(s)

M: 338e7821e3f84ed3124b4e3667d0a49d2e0f602f 127.0.0.1:7001

   slots:5461-10922 (5462 slots) master

   1 additional replica(s)

S: f60895e75326d4d890202ac46f68a5af390ace54 127.0.0.1:7004

   slots: (0 slots) slave

   replicates 338e7821e3f84ed3124b4e3667d0a49d2e0f602f

S: b9096b43e1d23d6ac9e79f49a45e8125b22b71be 127.0.0.1:7005

   slots: (0 slots) slave

   replicates 410e0ec3d0c88187fc9887232a233e1d26cf5a28

[OK] All nodes agree about slots configuration.

>>> Check for open slots...

>>> Check slots coverage...

[OK] All 16384 slots covered.

1.5     登陆集群

[root@izwz9gxsbwybwg9n5xdi47z redis-3.2.9]# /opt/redis-3.2.9/src/redis-cli -c -p 7000

127.0.0.1:7000> cluster info

cluster_state:ok

cluster_slots_assigned:16384

cluster_slots_ok:16384

cluster_slots_pfail:0

cluster_slots_fail:0

cluster_known_nodes:6

cluster_size:3

cluster_current_epoch:6

cluster_my_epoch:1

cluster_stats_messages_sent:302

cluster_stats_messages_received:302

127.0.0.1:7000>


1.6 测试验证
登录其中一个主节点
[root@izwz9gxsbwybwg9n5xdi47z src]# ./redis-cli -c -h 127.0.0.1 -p 7000
设置key值
127.0.0.1:7000> set name 'huangxueliang'
-> Redirected to slot [5798] located at 127.0.0.1:7001
OK

登录一个从节点get该key值,看集群是否同步，从下面的输出可以看出，集群已经同步.
[root@izwz9gxsbwybwg9n5xdi47z src]# ./redis-cli -c -h 127.0.0.1 -p 7003
127.0.0.1:7003> get name
-> Redirected to slot [5798] located at 127.0.0.1:7001
"huangxueliang"

-----------------------添加master新节点-------------------------
[root@izwz9gxsbwybwg9n5xdi47z redis-cluster]# mkdir 7006


进入7000目录,拷贝配置文件到准备新添加的节点目录7006
[root@izwz9gxsbwybwg9n5xdi47z 7000]# cp redis.conf ../7006/


进入到7006,修改配置文件
7006的配置文件
[root@izwz9gxsbwybwg9n5xdi47z 7006]# more redis.conf 
daemonize yes
pidfile /opt/redis-cluster/redis-7006.pid
port 7006
tcp-backlog 511
timeout 0
tcp-keepalive 0
loglevel notice
logfile "/opt/redis-cluster/log/redis-7006.log"
databases 16
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump-7006.rdb
dir /opt/redis-cluster/rbd
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
slave-priority 100
appendonly yes
appendfilename "redis-7006.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
lua-time-limit 5000
cluster-enabled yes
cluster-config-file /opt/redis-cluster/nodes/nodes-7006.conf
cluster-node-timeout 15000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-entries 512
list-max-ziplist-value 64
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
aof-rewrite-incremental-fsync yes
# Generated by CONFIG REWRITE
#masterauth "richinfo123"
#requirepass "richinfo123"
protected-mode no


启动7006
/opt/redis-3.2.9/src/redis-server /opt/redis-cluster/7006/redis.conf


添加主节点到集群
/opt/redis-3.2.9/src/redis-trib.rb add-node 127.0.0.1:7006 127.0.0.1:7000
第一个 ip:port 为新节点 
第二个 ip:port 是任意一个已经存在的节点 
新节点没有包含任何数据，也没有包含任何slot.
当集群需要将某个从节点升级为新的主节点时,这个新节点不会被选中,同时新的主节点因为没有包含任何slot,不参加选举和failover.


为新节点分配slot
[root@izwz9gxsbwybwg9n5xdi47z src]# ./redis-trib.rb reshard 127.0.0.1:7006
>>> Performing Cluster Check (using node 127.0.0.1:7006)
M: 2a2266ce2445f0e82ad58b58e5dd727db0357901 127.0.0.1:7006
   slots: (0 slots) master
   0 additional replica(s)
S: f60895e75326d4d890202ac46f68a5af390ace54 127.0.0.1:7004
   slots: (0 slots) slave
   replicates 338e7821e3f84ed3124b4e3667d0a49d2e0f602f
S: 0eca33579c9e9d3c470d591d0f8d709855432db5 127.0.0.1:7003
   slots: (0 slots) slave
   replicates bc6e5ffeb32d65b9b8d9cdc3c3a1997a2f3b3728
M: 338e7821e3f84ed3124b4e3667d0a49d2e0f602f 127.0.0.1:7001
   slots:5461-10922 (5462 slots) master
   1 additional replica(s)
S: b9096b43e1d23d6ac9e79f49a45e8125b22b71be 127.0.0.1:7005
   slots: (0 slots) slave
   replicates 410e0ec3d0c88187fc9887232a233e1d26cf5a28
M: 410e0ec3d0c88187fc9887232a233e1d26cf5a28 127.0.0.1:7002
   slots:10923-16383 (5461 slots) master
   1 additional replica(s)
M: bc6e5ffeb32d65b9b8d9cdc3c3a1997a2f3b3728 127.0.0.1:7000
   slots:0-5460 (5461 slots) master
   1 additional replica(s)
[OK] All nodes agree about slots configuration.
>>> Check for open slots...
>>> Check slots coverage...
[OK] All 16384 slots covered.
How many slots do you want to move (from 1 to 16384)? 500
根据提示选择要迁移的slot数量(ps:这里选择500)
What is the receiving node ID? 
这里输入7006节点的id:2a2266ce2445f0e82ad58b58e5dd727db0357901
Do you want to proceed with the proposed reshard plan (yes/no)? yes

----------添加新的Slave节点------------------
[root@izwz9gxsbwybwg9n5xdi47z redis-cluster]# mkdir 7007
进入7000目录,拷贝配置文件到准备新添加的节点目录7007
[root@izwz9gxsbwybwg9n5xdi47z 7000]# cp redis.conf ../7007/


进入7007目录,修改配置文件
7007配置文件
[root@izwz9gxsbwybwg9n5xdi47z 7007]# more redis.conf 
daemonize yes
pidfile /opt/redis-cluster/redis-7007.pid
port 7007
tcp-backlog 511
timeout 0
tcp-keepalive 0
loglevel notice
logfile "/opt/redis-cluster/log/redis-7007.log"
databases 16
save 900 1
save 300 10
save 60 10000
stop-writes-on-bgsave-error yes
rdbcompression yes
rdbchecksum yes
dbfilename dump-7007.rdb
dir /opt/redis-cluster/rbd
slave-serve-stale-data yes
slave-read-only yes
repl-diskless-sync no
repl-diskless-sync-delay 5
repl-disable-tcp-nodelay no
slave-priority 100
appendonly yes
appendfilename "redis-7007.aof"
appendfsync everysec
no-appendfsync-on-rewrite no
auto-aof-rewrite-percentage 100
auto-aof-rewrite-min-size 64mb
aof-load-truncated yes
lua-time-limit 5000
cluster-enabled yes
cluster-config-file /opt/redis-cluster/nodes/nodes-7007.conf
cluster-node-timeout 15000
slowlog-log-slower-than 10000
slowlog-max-len 128
latency-monitor-threshold 0
notify-keyspace-events ""
hash-max-ziplist-entries 512
hash-max-ziplist-value 64
list-max-ziplist-entries 512
list-max-ziplist-value 64
set-max-intset-entries 512
zset-max-ziplist-entries 128
zset-max-ziplist-value 64
hll-sparse-max-bytes 3000
activerehashing yes
client-output-buffer-limit normal 0 0 0
client-output-buffer-limit slave 256mb 64mb 60
client-output-buffer-limit pubsub 32mb 8mb 60
hz 10
aof-rewrite-incremental-fsync yes
# Generated by CONFIG REWRITE
#masterauth "richinfo123"
#requirepass "richinfo123"
protected-mode no


启动7007节点
/opt/redis-3.2.9/src/redis-server /opt/redis-cluster/7007/redis.conf


添加slave节点
./redis-trib.rb add-node --slave 127.0.0.1:7007 127.0.0.1:7006
127.0.0.1:7007 为slave节点,127.0.0.1:7006为刚新增的master节点

--------------------安装问题汇总-----------------------------------
1.bg_complex_data.tcl
"bg_complex_data [lindex $argv 0] [lindex $argv 1] [lindex $argv 2] [lindex $argv 3]"
    (file "tests/helpers/bg_complex_data.tcl" line 10)
Killing still running Redis server 21198
make: *** [test] Error 1

解决办法:
[root@localhost src]# vim ../tests/integration/replication-2.tcl 
start_server {tags {"repl"}} {
    start_server {} {
        test {First server should have role slave after SLAVEOF} {
            r -1 slaveof [srv 0 host] [srv 0 port]
            after 10000    #修改成10000
            s -1 role