分类： 架构设计与优化
2013-05-09 13:32:17
1.环境介绍
1.1.两台linux分别安装keepalivd+Nginx，功能如下：
keepalived提供nginx负载调度（HA）
nginx提供反向代理内网 172.18.0.0/24的web服务器，
iptables设置SNAT映射，实现同时实现172.18.0.0/24的服务器能通过nginx代理服务器上网
通过配置策略路由实现双链路访问，具体可参考
1.2 ip分别情况
|
服务器 |
网通ip |
电信ip |
网通负载ip |
电信负载ip |
|
Nginx1 |
61.49.23.23 |
220.181.45.46 |
61.49.23.25 |
220.181.45.48 |
|
Nginx2 |
61.49.23.24 |
220.181.45.47 |
||
|
网关 |
61.49.23.22 |
220.181.45.43 |
2. Nginx+keepalive安装配置
2.1.1 Keepalived安装
#yum -y install kernel-devel make gcc openssl-devel lftp libnl* popt*
#ln -s /usr/src/kernels/`uname -r`-`uname -m`/ /usr/src/linux
#tar zxvf keepalived-1.1.20.tar.gz
#cd keepalived-1.1.20
#./configure --prefix=/usr --sysconf=/etc --with-kernel-dir=/usr/src/kernels/2.6.18-348.4.1.el5-x86_64
Keepalived configuration
------------------------
Keepalived version : 1.1.20
Compiler : gcc
Compiler flags : -g -O2
Extra Lib : -lpopt -lssl -lcrypto
Use IPVS Framework : Yes ;注意编译时一定要支持lvs
IPVS sync daemon support : Yes
Use VRRP Framework : Yes
Use LinkWatch : No
Use Debug flags : No
#make
#make install
2.1.2 keepalived配置文件
global_defs {
notification_email {
changyz@bitauto.com
}
notification_email_from lvs1@bitautotech.com
smtp_server 127.0.0.1
smtp_connect_timeout 30
router_id LVS_MASTER
}
vrrp_sync_group VG1 {
group {
VI_1
VI_GATEWAY
}
}
vrrp_sync_group VG2 {
group {
VI_2
VI_GATEWAY
}
}
vrrp_instance VI_1 {
state MASTER ;备keepalived为BACKUP
interface eth0
virtual_router_id 56
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 111111
}
virtual_ipaddress {
61.49.23.25 ；网通负载地址
}
}
vrrp_instance VI_2 {
state MASTER ;备keepalived为BACKUP
interface eth1
virtual_router_id 57
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 111111
}
virtual_ipaddress {
220.181.45.48 ；电信负载地址
}
}
vrrp_instance VI_GATEWAY {
state MASTER
interface eth2
virtual_router_id 58
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 111111
}
virtual_ipaddress {
172.18.0.250
}
}
virtual_server 61.49.23.25 80 {
delay_loop 6
lb_algo wrr
lb_kind DR
#nat_mask 255.255.255.0
protocol TCP
persistence_timeout 300
real_server 61.49.23.23 80 {
weight 1
TCP_CHECK {
connect_timeout 3
}
}
real_server 61.49.23.24 80 {
weight 1
TCP_CHECK {
connect_timeout 3
}
}}
virtual_server 220.181.45.48 80 {
delay_loop 6
lb_algo wrr
lb_kind DR
#nat_mask 255.255.255.0
protocol TCP
persistence_timeout 300
real_server 220.181.45.46 80 {
weight 1
TCP_CHECK {
connect_timeout 3
}
}
real_server 220.181.45.47 80 {
weight 1
TCP_CHECK {
connect_timeout 3
}
}
2.2 Nginx配置部署
2.2.1 安装
#pcre-8.12
#./configure
#make && make install
#cd ../
#cd nginx-1.4.0
#./configure --user=nginx --group=nginx --prefix=/usr/local/nginx --with-http_stub_status_module --with-http_ssl_module
#make && make install
2.2.2 nginx配置
user nginx nginx;
worker_processes 16;
error_log /var/log/nginx_error.log crit;
pid /usr/local/nginx/nginx.pid;
worker_rlimit_nofile 65535;
events
{
use epoll;
worker_connections 65535;
}
http
{
include mime.types;
default_type application/octet-stream;
server_names_hash_bucket_size 128;
client_header_buffer_size 32k;
large_client_header_buffers 4 32k;
client_max_body_size 8m;
sendfile on;
tcp_nopush on;
keepalive_timeout 60;
tcp_nodelay on;
fastcgi_connect_timeout 300;
fastcgi_send_timeout 300;
fastcgi_read_timeout 300;
fastcgi_buffer_size 64k;
fastcgi_buffers 8 128k;
fastcgi_busy_buffers_size 128k;
fastcgi_temp_file_write_size 128k;
gzip on;
gzip_min_length 1k;
gzip_buffers 4 16k;
gzip_http_version 1.0;
gzip_comp_level 2;
gzip_types text/plain application/x-javascript text/css application/xml;
gzip_vary on;
log_format access '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" $http_x_forwarded_for';
include vhosts/upstream.conf; ；反向代理转发配置
include vhosts/test1.com.conf; ；虚拟主机配置文件
}
2.2.3代理转发配置文件upstream.conf
upstream test1 {
server 172.18.0.101:80; ;后端web服务器
}
2.2.4虚拟主机配置文件test1.com.conf
server
{
listen 80;
server_name
index index.html index.shtml index.php;
location / {
proxy_pass
proxy_redirect off;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
client_max_body_size 10m;
client_body_buffer_size 256k;
proxy_connect_timeout 60;
proxy_send_timeout 60;
proxy_read_timeout 60;
proxy_buffer_size 256k;
proxy_buffers 4 256k;
proxy_busy_buffers_size 256k;
proxy_temp_file_write_size 256k;
}
access_log /var/log/access.log access;
}
}
~
2.3. 网关及路由设置
2.3.1. 添加策略路由(以nginx1为例)
echo "251 eth0" >> /etc/iproute2/rt_tables
echo "252 eth1" >> /etc/iproute2/rt_tables
2.3.2. 策略路由规则
网通：
ip route flush table eth0
ip route add default via 61.49.23.22 dev eth0 src 61.49.23.23 table eth0 prio 50
ip rule add from 61.49.23.23 table eth0
电信：
ip route flush table eth1
ip route add default via 220.181.45.43 ev eth1 src 220.181.45.46 able eth1 prio 50
ip rule add from 220.181.45.46 table eth1
ip route add 172.18.0.0/24 dev eth2 scope link src 172.18.0.7 table eth0
ip route add 172.18.0.0/24 dev eth2 scope link src 172.18.0.7 table eth1
2.4 内网SNAT设置
/sbin/iptables -t nat -A POSTROUTING -s 172.18.0.0/255.255.255.0 -o eth1 -j SNAT --to-source 61.49.23.23；从eth0出去的ip映射为网通地址
/sbin/iptables -t nat -A POSTROUTING -s 172.18.0.0/255.255.255.0 -o eth1 -j SNAT --to-source 220.181.45.46 ；从eth1出去的ip映射为电信地址
3. web路由指向
172.18.0.0/24要访问外网，需要将网关指向172.18.0.250,通过nginx代理服务器上SNAT来访问外网。
