分类： 架构设计与优化
2013-02-11 11:29:12
Centos5.3以上内核默认支持lvs功能，在本文中为了方便安装部署lvs使用了centos5.8系统。
具体安装步骤如下所示：
网络地址规划：
vip: 10.0.211.30
master 10.0.211.5 backup:10.0.211.4
realServer 1:10.0.211.2 realServer 2:10.0.211.3 realServer 3:10.0.211.4
realServer 4:10.0.211.12
一、 检查kernel是否已经支持ipvs模块
[root@server conf]# modprobe -l|grep ipvs
/lib/modules/2.6.18-308.el5xen/kernel/net/ipv4/ipvs/ip_vs.ko
/lib/modules/2.6.18-308.el5xen/kernel/net/ipv4/ipvs/ip_vs_dh.ko
/lib/modules/2.6.18-308.el5xen/kernel/net/ipv4/ipvs/ip_vs_ftp.ko
/lib/modules/2.6.18-308.el5xen/kernel/net/ipv4/ipvs/ip_vs_lblc.ko
/lib/modules/2.6.18-308.el5xen/kernel/net/ipv4/ipvs/ip_vs_lblcr.ko
/lib/modules/2.6.18-308.el5xen/kernel/net/ipv4/ipvs/ip_vs_lc.ko
/lib/modules/2.6.18-308.el5xen/kernel/net/ipv4/ipvs/ip_vs_nq.ko
/lib/modules/2.6.18-308.el5xen/kernel/net/ipv4/ipvs/ip_vs_rr.ko
/lib/modules/2.6.18-308.el5xen/kernel/net/ipv4/ipvs/ip_vs_sed.ko
/lib/modules/2.6.18-308.el5xen/kernel/net/ipv4/ipvs/ip_vs_sh.ko
/lib/modules/2.6.18-308.el5xen/kernel/net/ipv4/ipvs/ip_vs_wlc.ko
/lib/modules/2.6.18-308.el5xen/kernel/net/ipv4/ipvs/ip_vs_wrr.ko
显示如上信息表示kernel已经支持lvs功能。
二、 安装keepalived
1、 解压
tar –zxvf keepalived-1.2.2.tar.gz
2、 配置环境
#cd /ha/keepalived-1.2.2
#./confiure
3、 编译安装
#make && make install
4、 配置成系统服务
cp /usr/local/etc/rc.d/init.d/keepalived /etc/init.d/
cp /usr/local/etc/sysconfig/keepalived /etc/sysconfig/
mkdir /etc/keepalived/
cp /usr/local/etc/keepalived/keepalived.conf /etc/keepalived/
cp /usr/local/sbin/keepalived /usr/sbin/
5、 启动和停止HA服务
service keepalived start
service keepalived stop
6、 检查HA的运行状态
service keepalived status
7、 配置slave HA文件
bal_defs {
notification_email {
fengzhanhai@allook.cn
}
notification_email_from fengzhanhai@allook.cn
smtp_server 202.102.188.42
smtp_connect_timeout 30
router_id LVS1
}
vrrp_sync_group test {
group {
loadbalance
}
}
vrrp_instance loadbalance {
state BACKUP
interface eth0
lvs_sync_daemon_inteface eth0
virtual_router_id 55
#此处需要注意如果当前环境已经存在其他lvs需要修改默认的routeid值
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.0.211.30 dev eth0 label eth0:1
}
}
virtual_server 10.0.211.30 80 {
delay_loop 6
lb_algo rr
lb_kind DR
persistence_timeout 20
protocol TCP
sorry_server 10.0.211.12 80
real_server 10.0.211.7 80 {
weight 3
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
}
8、 配置master HA文件
global_defs {
notification_email {
fengzhanhai@allook.cn
}
notification_email_from fengzhanhai@allook.cn
smtp_server 202.102.188.42
smtp_connect_timeout 30
router_id LVS1
}
vrrp_sync_group test {
group {
loadbalance
}
}
vrrp_instance loadbalance {
state MASTER
interface eth0
lvs_sync_daemon_inteface eth0
virtual_router_id 55
priority 180
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
10.0.211.30 dev eth0 label eth0:1}
}
virtual_server 10.0.211.30 80 {
delay_loop 6
lb_algo rr
lb_kind DR
persistence_timeout 20
protocol TCP
sorry_server 10.0.211.12 80
real_server 10.0.211.7 80 {
weight 3
TCP_CHECK {
connect_timeout 3
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
}
三、 安装lvs模块ipvsadm
1、 解压tar –zxvf ipvsadm-1.24.tar.gz
2、 编译
#cd ipvsadm-1.24
#make
报错如下所示：
在包含自 libipvs.c：23 的文件中:
libipvs.h:14:23: 错误：net/ip_vs.h：没有那个文件或目录
In file included from libipvs.c:23:
libipvs.h:119: 错误：expected ‘)’ before ‘fwmark’
libipvs.c:27: 错误：字段 ‘svc’ 的类型不完全
libipvs.c:28: 错误：字段 ‘dest’ 的类型不完全
libipvs.c: In function ‘ipvs_init’:
libipvs.c:40: 错误：‘sizeof’ 不能用于不完全的类型 ‘struct ip_vs_getinfo’
libipvs.c:44: 错误：‘IP_VS_SO_GET_INFO’ 未声明 (在此函数内第一次使用)
libipvs.c:44: 错误：(即使在一个函数内多次出现，每个未声明的标识符在其
libipvs.c:44: 错误：所在的函数内只报告一次。)
libipvs.c: In function ‘ipvs_getinfo’:
libipvs.c:56: 错误：‘sizeof’ 不能用于不完全的类型 ‘struct ip_vs_getinfo’
解决方法：
1、#modprobe ip_vs //加载ip_vs模块
2、提示找不到net/ip_vs.h：没有那个文件或目录
find / -name 'ip_vs*'
cp /ha/keepalived-1.2.2/keepalived/libipvs-2.6/ip_vs.h /usr/include/net
再次make时提示如下错误信息：
make -C libipvs
make[1]: Entering directory `/ha/ipvsadm-1.24/libipvs'
gcc -Wall -Wunused -Wstrict-prototypes -g -O2 -I/usr/src/linux/include -DHAVE_NET_IP_VS_H -c -o libipvs.o libipvs.c
libipvs.c: In function ‘ipvs_cmp_services’:
libipvs.c:197: 错误：初始化时类型不兼容
libipvs.c:197: 错误：初始化时类型不兼容
libipvs.c: In function ‘ipvs_cmp_dests’:
libipvs.c:243: 错误：初始化时类型不兼容
libipvs.c:243: 错误：初始化时类型不兼容
libipvs.c: In function ‘ipvs_get_service’:
libipvs.c:272: 错误：赋值时类型不兼容
make[1]: *** [libipvs.o] 错误 1
make[1]: Leaving directory `/ha/ipvsadm-1.24/libipvs'
make: *** [libs] 错误 2
使用uname -r查看内核2.6.18-128.el5-i686后建立软连接2.6.18-128.el5-i686
#ln -s /usr/src/kernels/2.6.18-128.el5-i686/ /usr/src/linux
依然提示错误相同错误
这时我们需要修改c程序makefile文件了
#cd /ha/ipvsadm-1.24/libipvs
#vi Makefile文件
#修改当前目录和libipvs目录中的 Makefile的：INCLUDE = -I/usr/src/linux/include -I.. -I.为：INCLUDE = -I/usr/src/kernels/2.6.9-22.EL-i686/include -I..
3、 最简单的方法使用yum安装
#yum install ipvsadm
四、 配置ipvsadm
1、 配置lvs在调度服务器上配置realserver及vip等资源
1） 添加虚拟server
#ipvsadm –A –t 10.0.211.30:80 –s wlc
#向lvs中添加一台基于tcp协议开放web服务的虚拟主机，调度策略为按权重进行调度
2） 添加realServer负载服务器
#ipvsadm –a –t 10.0.211.30:80 –r 10.0.211.2:80 –g
#ipvsadm –a –t 10.0.211.30:80 –r 10.0.211.3:80 –g
#以上表示为虚拟服务器10.0.211.30添加了两台真实服务器并设置为DR直接路由模式
3） 删除一个真实web
ipvsadm -d -t 10.0.211.30:80 -r 10.0.211.2:80
4） 删除一个虚拟Server
[root@server /]# ipvsadm -D -t 10.0.211.30:80
2、 在realServer中添加路由到lo：0设备上使得lvs起作用
#!/bin/bash
VIP=10.0.211.40
. /etc/rc.d/init.d/functions
case "$1" in
start)
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
ifconfig lo:0 $VIP broadcast $VIP netmask 255.255.255.255 up
/sbin/route add -host $VIP dev lo:0
sysctl -p > /dev/null 2>&1
echo "realserver start OK"
;;
stop)
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
ifconfig lo:0 down
/sbin/route del $VIP > /dev/null 2>&1
echo "realserver stoped"
;;
*)
echo "Usage:$0 {start|stop}"
exit 1
esac
exit 0
五、 ipvsadm常用参数概览
ipvsadm常用参数配置表如下所示：
六、 ipvsadm用法示例
ipvsadm命令的用法:
ipvsadm v1.24 2005/12/10 (compiled with popt and IPVS v1.2.1)
Usage:
ipvsadm -A|E -t|u|f service-address [-s scheduler] [-p [timeout]] [-M netmask]
ipvsadm -D -t|u|f service-address
ipvsadm -C
ipvsadm -R
ipvsadm -S [-n]
ipvsadm -a|e -t|u|f service-address -r server-address [options]
ipvsadm -d -t|u|f service-address -r server-address
ipvsadm -L|l [options]
ipvsadm -Z [-t|u|f service-address]
ipvsadm --set tcp tcpfin udp
ipvsadm --start-daemon state [--mcast-interface interface] [--syncid sid]
ipvsadm --stop-daemon state
ipvsadm -h
命令及其含义：
--add-service -A add virtual service with options
--edit-service -E edit virtual service with options
--delete-service -D delete virtual service
--clear -C clear the whole table
--restore -R restore rules from stdin
--save -S save rules to stdout
--add-server -a add real server with options
--edit-server -e edit real server with options
--delete-server -d delete real server
--list -L|-l list the table
--zero -Z zero counters in a service or all services
--set tcp tcpfin udp set connection timeout values
--start-daemon start connection sync daemon
--stop-daemon stop connection sync daemon
--help -h display this help message
