基于云端虚拟机的LVS/DR+Keepalived+nginx的高可用集群架构配置
最近，公司要我部署一个集群架构，选来选取还是选择了大家都很熟悉的基于DR的LVS+Keepalived做负载分发，然后使用轻量级的nginx做中间代理层，这里呢先暂时实现一个简单web应用功能，对于代理功能和后续的web层部署、数据层安排将择机更新！
首先看一下集群框架：
1）LVS1：10.124.20.248&&LVS2：10.124.20.249
cd /usr/local/src/
tar xf ipvsadm-1.24.tar.gz
tar xf keepalived-1.1.20.tar.gz
ipvsadm编译安装
ln -s /usr/src/kernels/2.6.32-279.el6.x86_64/ /usr/src/linux #创建软连接
cd /usr/local/src/ipvsadm-1.24
make && make install
#/sbin/ipvsadm -v #检测ipvsadm是否安装成功(或者：#watch ipvsadm –ln)
ipvsadm v1.24 2005/12/10 (compiled with getopt_long and IPVS v1.2.1)
将ipvsadm设置成开机自启动服务
chkconfig –add keepalived
chkconfig keepalived on
Keepalived编译安装
/usr/local/src/keepalived-1.1.20
./configure –prefix=/usr/local/keepalived –with-kernel-dir=/usr/src/kernels/2.6.32-279.el6.x86_64
返回信息：
Keepalived configuration
————————
Keepalived version : 1.1.20
Compiler : gcc
Compiler flags : -g -O2
Extra Lib : -lpopt -lssl -lcrypto
Use IPVS Framework : Yes
IPVS sync daemon support : Yes
Use VRRP Framework : Yes
Use Debug flags : No
make && make install
返回信息：
install -d /usr/local/keepalived/bin
install -m 755 ../bin/genhash /usr/local/keepalived/bin/
install -d /usr/local/keepalived/share/man/man1
install -m 644 ../doc/man/man1/genhash.1 /usr/local/keepalived/share/man/man1
将Keepalived做成服务：
cd /usr/local/keepalived/
cp etc/rc.d/init.d/keepalived /etc/rc.d/init.d/
cp etc/sysconfig/keepalived /etc/sysconfig/
mkdir /etc/keepalived
cp etc/keepalived/keepalived.conf /etc/keepalived/
cp sbin/keepalived /usr/sbin/
2） 修改Keepalived.conf配置文件
#cat /etc/keepalived/keepalived.conf #主LVS1：10.124.20.248
! Configuration File for keepalived
global_defs {
notification_email {
qingbo.song@apicloud.com
}
notification_email_from qingbo.song@apicloud.com
smtp_server 127.0.0.1
# smtp_connect_timeout 30
router_id LVS_DEVEL
}
vrrp_sync_group VGM {
group {
VI_1
}
notify_master “/opt/shell/arp.sh”
notify_backup “/opt/shell/arp.sh”
}
vrrp_script chk_nginx {
script “/opt/shell/check_nginx.sh”
interval 1
weight -2
}
vrrp_script chk_nginx2 {
script “/opt/shell/check_nginx2.sh”
interval 1
weight -2
}
vrrp_script chk_nginx3 {
script “/opt/shell/check_nginx3.sh”
interval 1
weight -2
}
vrrp_instance VI_1 {
state MASTER
interface eth0
virtual_router_id 51
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass yzkjTest
}
track_script {
chk_nginx
chk_nginx2
chk_nginx3
}
virtual_ipaddress {
10.124.20.20
}
}
virtual_server 10.124.20.20 80 {
delay_loop 5
lb_algo sh
lb_kind DR
persistence_timeout 60
protocol TCP
real_server 10.124.20.250 80 {
weight 1
TCP_CHECK {
connect_timeout 10
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
real_server 10.124.20.251 80 {
weight 1
TCP_CHECK {
connect_timeout 10
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
real_server 10.124.20.252 80 {
weight 1
TCP_CHECK {
connect_timeout 10
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
}
#cat /etc/keepalived/keepalived.conf #备LVS1：10.124.20.249
! Configuration File for keepalived
global_defs {
notification_email {
qingbo.song@apicloud.com
}
notification_email_from qingbo.song@apicloud.com
smtp_server 127.0.0.1
# smtp_connect_timeout 30
router_id LVS_DEVEL
}
vrrp_sync_group VGM {
group {
VI_1
}
notify_master “/opt/shell/arp.sh”
notify_backup “/opt/shell/arp.sh”
}
vrrp_script chk_nginx {
script “/opt/shell/check_nginx.sh”
interval 1
weight -2
}
vrrp_script chk_nginx2 {
script “/opt/shell/check_nginx2.sh”
interval 1
weight -2
}
vrrp_script chk_nginx3 {
script “/opt/shell/check_nginx3.sh”
interval 1
weight -2
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
virtual_router_id 51
priority 99
advert_int 1
authentication {
auth_type PASS
auth_pass yzkjTest
}
track_script {
chk_nginx
chk_nginx2
chk_nginx3
}
virtual_ipaddress {
10.124.20.20
}
}
virtual_server 10.124.20.20 80 {
delay_loop 5
lb_algo sh
lb_kind DR
persistence_timeout 60
protocol TCP
real_server 10.124.20.250 80 {
weight 1
TCP_CHECK {
connect_timeout 10
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
real_server 10.124.20.251 80 {
weight 1
TCP_CHECK {
connect_timeout 10
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
real_server 10.124.20.252 80 {
weight 1
TCP_CHECK {
connect_timeout 10
nb_get_retry 3
delay_before_retry 3
connect_port 80
}
}
}
==============================================================
相关检测脚本：
#cat /opt/shell/arp.sh
#!/bin/bash
VIP=10.124.20.20
GATEWAY=10.124.20.1
/sbin/arping -I eth0 -c 5 -s $VIP $GATEWAY &>/dev/null
#cat /opt/shell/check_nginx.sh
!/bin/bash
url=”http://10.124.20.250″
status=$(/usr/bin/curl -s –head “$url” | awk ‘/HTTP/ {print $2}’)
if [ “$status” != “200” ]; then
/opt/shell/arp.sh start
fi
#cat /opt/shell/check_nginx2.sh
!/bin/bash
url=”http://10.124.20.251″
status=$(/usr/bin/curl -s –head “$url” | awk ‘/HTTP/ {print $2}’)
if [ “$status” != “200” ]; then
/opt/shell/arp.sh start
fi
#cat /opt/shell/check_nginx3.sh
!/bin/bash
url=”http://10.124.20.252″
status=$(/usr/bin/curl -s –head “$url” | awk ‘/HTTP/ {print $2}’)
if [ “$status” != “200” ]; then
/opt/shell/arp.sh start
fi
===============================================================
3）Keepalived服务启动（ifconfig eth0:1 10.124.20.248 netmask 255.255.255.255 up）
service keepalived start #服务启动
#ps -aux|grep keepalived #检测服务启动状态
（/usr/local/keepalived/sbin/keepalived -D -f /etc/keepalived/keepalived.conf -D 显示在日志记录 -f 指定配置文件目录）
Warning: bad syntax, perhaps a bogus ‘-‘? See /usr/share/doc/procps-3.2.8/FAQ
root 21205 0.0 0.0 36776 732 ? Ss 17:06 0:00 keepalived -D
root 21207 0.0 0.1 38880 1936 ? S 17:06 0:00 keepalived -D
root 21208 0.0 0.1 38880 1232 ? S 17:06 0:00 keepalived -D
root 21301 0.0 0.0 103240 840 pts/0 S+ 17:07 0:00 grep keepalived
4）#设置开机启动服务
（##echo “/usr/local/keepalived/sbin/keepalived -D -f /etc/keepalived/keepalived.conf” >> /etc/rc.d/rc.local）
chkconfig –add keepalived
chkconfig keepalived on
service iptables stop
5）添加防火墙规则：
iptables -A INPUT -m state –state NEW -m tcp -p tcp –dport 80 -j ACCEPT #使80端口提供对外服务
iptables -A INPUT -d 224.0.0.0/8 -j ACCEPT
iptables -A INPUT -p vrrp -j ACCEPT #基于DR模式，当用户发出请求后，只有DR响应ARP广播包，允许vrrp虚拟路由器冗余协议
/etc/rc.d/init.d/iptables save #保存规则到iptables文件，使重启后生效）
6）开启转发功能；
#vim /etc/sysctl.conf
net.ipv4.ip_forward = 1
#sysctl -p #使转发能够生效
三、nginx服务器的搭建
1）安装pcre-8.36.tar.gz
cd /usr/local/src
tar xf pcre-8.36.tar.gz
cd pcre-8.36
./configure –prefix=/usr/local/pcre
make && make install
2）安装nginx-1.5.1.tar.gz
cd /usr/local/src
tar xf nginx-1.5.1.tar.gz
cd nginx-1.5.1
你在雨中行走，你从不打伞，你有自己的天空，它从不下雨。
