lvs+keepalived

keepalived做HA对后端有健康检查，可通过发送邮件告知运维人员。

注意：主备机的id必须一致，master的优先级必须大于备机

 

实验环境：rhel6.5 selinux and iptables disabled

HA主机：     192.168.2.138     192.168.2.135

real server: 192.168.2.116     192.168.2.160

virtual server:192.168.2.252

下载软件包：keepalived-1.2.12

[root@server38 keepalived-1.2.12]# ./configure --prefix=/usr/local/keepalived

 

configure: error: 

  !!! OpenSSL is not properly installed on your system. !!!

  !!! Can not include OpenSSL headers files.            !!![root@server38 keepalived-1.2.12]# yum install openssl-devel -y

 

[root@server38 keepalived-1.2.12]# ./configure --prefix=/usr/local/keepalived

IPVS use libnl           : No

Use VRRP Framework       : Yes

Use VRRP VMAC            : Yes

SNMP support             : No

SHA1 support             : No

Use Debug flags          : No

[root@server38 keepalived-1.2.12]# yum install libnl-devel -y

 

[root@server38 keepalived-1.2.12]# ./configure --prefix=/usr/local/keepalived

Use IPVS Framework       : Yes

IPVS sync daemon support : Yes

IPVS use libnl           : Yes

Use VRRP Framework       : Yes

Use VRRP VMAC            : Yes

看到5个yes即ok

 

编译且安装

[root@server38 keepalived-1.2.12]# make && make install

 

 

[root@server38 keepalived]# cat keepalived.conf 

! Configuration File for keepalived

 

global_defs {

   notification_email {

     root@localhost#接收警报的 email 地址,可以添加多个

   }

   notification_email_from [e-mail] #设置邮件的发送地址

   smtp_server 127.0.0.1 #设置 smtp server 地址

   smtp_connect_timeout 30 #设置连接 smtp 服务器超时时间

   router_id LVS_DEVEL #load balancer 的标识 ID,用于 email 警报

}

 

vrrp_instance VI_1 {

    state MASTER

    interface eth0

    virtual_router_id 38  #与备机的id必须一致

    priority 100  #优先级必须大于备机

    advert_int 1

    authentication {

        auth_type PASS

        auth_pass 1111

    }

    virtual_ipaddress {

        192.168.2.252  #virtual server

    }

}

 

virtual_server 192.168.2.252 80 {

    delay_loop 6

    lb_algo rr

    lb_kind DR

   # persistence_timeout 50

    protocol TCP

 

    real_server 192.168.2.116 80 {

        weight 1

TCP_CHECK{

            connect_timeout 3

            nb_get_retry 3

            delay_before_retry 3

        }

real_server 192.168.2.160 80 {

        weight 1

        TCP_CHECK{

            connect_timeout 3

            nb_get_retry 3

            delay_before_retry 3

        }

 

    }

}

 

 

[root@server38 local]# scp -r keepalived/ 192.168.2.135:/usr/local/

 

[root@server35 keepalived]# ln -s /usr/local/keepalived/etc/rc.d/init.d/keepalived /etc/rc.d/init.d/

[root@server35 keepalived]# ln -s /usr/local/keepalived/etc/sysconfig/keepalived /etc/sysconfig/

[root@server35 keepalived]# ln -s /usr/local/keepalived/etc/keepalived/ /etc/

[root@server35 keepalived]# ln -s /usr/local/keepalived/sbin/keepalived  /sbin/

[root@server35 keepalived]# cat keepalived.conf 

! Configuration File for keepalived

 

global_defs {

   notification_email {

root@localhost#接收警报的 email 地址,可以添加多个

 

   }

   notification_email_from [e-mail]#设置邮件的发送地址

 

   smtp_server 127.0.0.1#设置 smtp server 地址

   smtp_connect_timeout 30#设置连接 smtp 服务器超时时间

   router_id LVS_DEVEL#load balancer 的标识 ID,用于 email 警报

 

}

 

vrrp_instance VI_1 {

    state BACKUP     #备机BACKUP,此状态是由 priority 的值来决定的,若当前master的priority 的值小于备机的值,那么将会失去 MASTER 状态

 

    interface eth0

    virtual_router_id 38

    priority 50

    advert_int 1

    authentication {

        auth_type PASS

        auth_pass 1111

    }

    virtual_ipaddress {

        192.168.2.252

    }

}

 

virtual_server 192.168.2.252 80 {

    delay_loop 6

    lb_algo rr

    lb_kind DR

   # persistence_timeout 50

    protocol TCP

 

    real_server 192.168.2.116 80 {

        weight 1

TCP_CHECK{

            connect_timeout 3

            nb_get_retry 3

            delay_before_retry 3

        }

real_server 192.168.2.160 80 {

        weight 1

        TCP_CHECK{

            connect_timeout 3

            nb_get_retry 3

            delay_before_retry 3

        }

 

    }

}

 

[root@server16 ~]# yum install arptables_jf

[root@server16 ~]# arptables -A IN -d 192.168.2.252 -j DROP

[root@server16 ~]# arptables -A OUT -s 192.168.2.252 -j mangle --mangle-ip-s 192.168.0.116

[root@server16 ~]# /etc/init.d/arptables_jf save

[root@server16 ~]# ifconfig eth0:1 192.168.2.252 netmask 255.255.255.255 up

[root@server16 ~]# echo server16.example.com >/var/www/html/index.html

[root@server16 ~]# /etc/init.d/httpd start

 

[root@server60 ~]#  yum install arptables_jf -y

[root@server60 ~]# arptables -A IN -d 192.168.2.252 -j DROP

[root@server60 ~]# arptables -A OUT -s 192.168.2.252 -j mangle --mangle-ip-s 192.168.0.160

[root@server60 ~]# /etc/init.d/arptables_jf save

[root@server60 ~]# ifconfig  eth0:1 192.168.2.252 netmask 255.255.255.255 up（此处的子网掩码表示不对外开放）

[root@server60 ~]# echo server60.example.com >/var/www/html/index.html

[root@server60 ~]# /etc/init.d/httpd start

 

 

[root@server38 local]# /etc/init.d/keepalived start

Starting keepalived:                                       [  OK  ]

[root@server38 local]# tail -f /var/log/messages

May 31 07:17:05 localhost Keepalived_healthcheckers[11443]: Configuration is using : 11339 Bytes

May 31 07:17:05 localhost Keepalived_vrrp[11444]: Using LinkWatch kernel netlink reflector...

May 31 07:17:05 localhost Keepalived_vrrp[11444]: VRRP sockpool: [ifindex(2), proto(112), unicast(0), fd(10,11)]

May 31 07:17:05 localhost Keepalived_healthcheckers[11443]: Using LinkWatch kernel netlink reflector...

May 31 07:17:06 localhost Keepalived_vrrp[11444]: VRRP_Instance(VI_1) Transition to MASTER STATE

May 31 07:17:07 localhost Keepalived_vrrp[11444]: VRRP_Instance(VI_1) Entering MASTER STATE

May 31 07:17:07 localhost Keepalived_vrrp[11444]: VRRP_Instance(VI_1) setting protocol VIPs.

May 31 07:17:07 localhost Keepalived_healthcheckers[11443]: Netlink reflector reports IP 192.168.2.252 

[root@server38 local]# ip addr show

1: lo:  mtu 16436 qdisc noqueue state UNKNOWN 

    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00

    inet 127.0.0.1/8 scope host lo

    inet6 ::1/128 scope host 

       valid_lft forever preferred_lft forever

2: eth0:  mtu 1500 qdisc pfifo_fast state UP qlen 1000

    link/ether 52:54:00:61:4e:92 brd ff:ff:ff:ff:ff:ff

    inet 192.168.2.138/24 brd 192.168.2.255 scope global eth0

    inet 192.168.2.252/32 scope global eth0

    inet6 fe80::5054:ff:fe61:4e92/64 scope link 

       valid_lft forever preferred_lft forever

 测试：

刷新出现轮询界面即可

 检测keepalived对后端的检查状况：

[root@server16 ~]# /etc/init.d/httpd stop

刷新界面，一直出现server60的测试页面，并且无其他报错则证明keepalived对后端具有健康检查

 

查看邮件：

[root@server38 keepalived]# mail

Heirloom Mail version 12.4 7/29/08.  Type ? for help.

"/var/spool/mail/root": 6 messages 5 new 6 unread

 U  1 Mail Delivery System  Sat May 31 07:32  73/2357  "Undelivered Mail Returned to Sender"

>N  2 keepalived@server38.  Sat May 31 08:09  17/645   "[LVS_DEVEL] Realserver [192.168.2.160]:80 - DOWN"

 N  3 keepalived@server38.  Sat May 31 08:11  17/645   "[LVS_DEVEL] Realserver [192.168.2.160]:80 - DOWN"

 N  4 keepalived@server38.  Sat May 31 08:13  17/645   "[LVS_DEVEL] Realserver [192.168.2.160]:80 - DOWN"

 N  5 keepalived@server38.  Sat May 31 08:14  17/645   "[LVS_DEVEL] Realserver [192.168.2.160]:80 - DOWN"

 N  6 keepalived@server38.  Sat May 31 08:15  17/644   "[LVS_DEVEL] Realserver [192.168.2.160]:80 - UP"

 

添加ftp服务：

ftp服务必须加persistent选项

[root@server16 ~]# yum install vsftpd -y

[root@server60 ~]# yum install vsftpd -y

 

编辑测试页面：

[root@server16 ~]# touch /var/ftp/server16

[root@server60 ~]# touch /var/ftp/server60

 

[root@server16 ~]# /etc/init.d/vsftpd start

[root@server60 ~]# /etc/init.d/vsftpd start

 

在主备LB上添加如下内容：

[root@server38 keepalived]# cat keepalived.conf 

virtual_server 192.168.2.252 21 {

    delay_loop 3

    lb_algo rr

    lb_kind DR

    persistence_timeout 100

    protocol TCP

 

    real_server 192.168.2.116 21 {

        weight 1

        TCP_CHECK {

            connect_timeout 3

            nb_get_retry 3

            delay_before_retry 3

        }

    }

 

    real_server 192.168.2.160 21 {

        weight 1

        TCP_CHECK {

            connect_timeout 3

            nb_get_retry 3

            delay_before_retry 3

        }

 

    }

}

[root@server38 keepalived]# /etc/init.d/keepalived stop

[root@server35 keepalived]# /etc/init.d/keepalived stop

[root@server38 keepalived]# /etc/init.d/keepalived start

[root@server35 keepalived]# /etc/init.d/keepalived start

 

[root@server38 keepalived]# ipvsadm -L

IP Virtual Server version 1.2.1 (size=4096)

Prot LocalAddress:Port Scheduler Flags

  -> RemoteAddress:Port           Forward Weight ActiveConn InActConn

TCP  192.168.2.252:ftp rr persistent 100

  -> 192.168.2.116:ftp            Route   1      0          1         

  -> 192.168.2.160:ftp            Route   1      0          0         

TCP  192.168.2.252:http rr

  -> 192.168.2.116:http           Route   1      0          0         

  -> 192.168.2.160:http           Route   1      0          0  

 

测试：

[root@cun Desktop]# lftp 192.168.2.252

lftp 192.168.2.252:~> ls

drwxr-xr-x    2 0        0            4096 Feb 12  2013 pub

-rw-r--r--    1 0        0               0 May 31 00:08 server16

 

等待100秒，再次访问vip的21端口时，会出现以下内容：

[root@cun Desktop]# lftp 192.168.2.252

lftp 192.168.2.252:~> ls

drwxr-xr-x    2 0        0            4096 Feb 12  2013 pub

-rw-r--r--    1 0        0               0 May 31 00:08 server60