主机规划;
ha1 eth0 172.16.100.71 gw 172.16.100.1
ha2 eth0 172.16.100.72 gw 172.16.100.1
vip: 172.16.100.70
编译安装nginx-1.5.4(两个节点都要做)
[root@ha1 ~]# yum groupinstall "Development Tools" -y [root@ha1 ~]# yum install openssl-devel pcre-devel -y [root@ha1 ~]# groupadd -r nginx [root@ha1 ~]# useradd -r -g nginx nginx [root@ha1 ~]# tar xvf nginx-1.5.4.tar.gz [root@ha1 ~]# cd nginx-1.5.4 [root@ha1 nginx-1.5.4]# ./configure \ > --prefix=/usr \ > --sbin-path=/usr/sbin/nginx \ > --conf-path=/etc/nginx/nginx.conf \ > --error-log-path=/var/log/nginx/error.log \ > --http-log-path=/var/log/nginx/access.log \ > --pid-path=/var/run/nginx/nginx.pid \ > --lock-path=/var/lock/nginx.lock \ > --user=nginx \ > --group=nginx \ > --with-http_ssl_module \ > --with-http_flv_module \ > --with-http_stub_status_module \ > --with-http_gzip_static_module \ > --http-client-body-temp-path=/var/tmp/nginx/client/ \ > --http-proxy-temp-path=/var/tmp/nginx/proxy/ \ > --http-fastcgi-temp-path=/var/tmp/nginx/fcgi/ \ > --http-uwsgi-temp-path=/var/tmp/nginx/uwsgi \ > --http-scgi-temp-path=/var/tmp/nginx/scgi \ > --with-pcre [root@ha1 nginx-1.5.4]# make && make install
为nginx提供SysV init脚本
[root@ha1 ~]# vim /etc/rc.d/init.d/nginx
[root@ha1 ~]# cat /etc/rc.d/init.d/nginx
#!/bin/sh
#
# nginx - this script starts and stops the nginx daemon
#
# chkconfig: - 85 15
# description: Nginx is an HTTP(S) server, HTTP(S) reverse \
# proxy and IMAP/POP3 proxy server
# processname: nginx
# config: /etc/nginx/nginx.conf
# config: /etc/sysconfig/nginx
# pidfile: /var/run/nginx.pid
# Source function library.
. /etc/rc.d/init.d/functions
# Source networking configuration.
. /etc/sysconfig/network
# Check that networking is up.
[ "$NETWORKING" = "no" ] && exit 0
nginx="/usr/sbin/nginx"
prog=$(basename $nginx)
NGINX_CONF_FILE="/etc/nginx/nginx.conf"
[ -f /etc/sysconfig/nginx ] && . /etc/sysconfig/nginx
lockfile=/var/lock/subsys/nginx
make_dirs() {
# make required directories
user=`nginx -V 2>&1 | grep "configure arguments:" | sed 's/[^*]*--user=\([^ ]*\).*/\1/g' -`
options=`$nginx -V 2>&1 | grep 'configure arguments:'`
for opt in $options; do
if [ `echo $opt | grep '.*-temp-path'` ]; then
value=`echo $opt | cut -d "=" -f 2`
if [ ! -d "$value" ]; then
# echo "creating" $value
mkdir -p $value && chown -R $user $value
fi
fi
done
}
start() {
[ -x $nginx ] || exit 5
[ -f $NGINX_CONF_FILE ] || exit 6
make_dirs
echo -n $"Starting $prog: "
daemon $nginx -c $NGINX_CONF_FILE
retval=$?
echo
[ $retval -eq 0 ] && touch $lockfile
return $retval
}
stop() {
echo -n $"Stopping $prog: "
killproc $prog -QUIT
retval=$?
echo
[ $retval -eq 0 ] && rm -f $lockfile
return $retval
}
restart() {
configtest || return $?
stop
sleep 1
start
}
reload() {
configtest || return $?
echo -n $"Reloading $prog: "
killproc $nginx -HUP
RETVAL=$?
echo
}
force_reload() {
restart
}
configtest() {
$nginx -t -c $NGINX_CONF_FILE
}
rh_status() {
status $prog
}
rh_status_q() {
rh_status >/dev/null 2>&1
}
case "$1" in
start)
rh_status_q && exit 0
$1
;;
stop)
rh_status_q || exit 0
$1
;;
restart|configtest)
$1
;;
reload)
rh_status_q || exit 7
$1
;;
force-reload)
force_reload
;;
status)
rh_status
;;
condrestart|try-restart)
rh_status_q || exit 0
;;
*)
echo $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|reload|force-reload|configtest}"
exit 2
esac[root@ha1 ~]# chmod +x /etc/rc.d/init.d/nginx [root@ha1 ~]# chkconfig --add nginx [root@ha1 ~]# chkconfig nginx on
配置Nginx:
[root@ha1 ~]# cp /etc/nginx/nginx.conf{,.bak}
[root@ha1 ~]# egrep -v "^#|^$|^[[:space:]]+#" /etc/nginx/nginx.conf
worker_processes 2;
events {
worker_connections 10240;
}
http {
include mime.types;
default_type application/octet-stream;
sendfile on;
keepalive_timeout 65;
upstream http-shop {
server 172.16.100.42 weight=1 max_fails=2 fail_timeout=2;
server 172.16.100.44 weight=1 max_fails=2 fail_timeout=2;
server 127.0.0.1:8080 backup;
}
upstream https-shop {
ip_hash;
server 172.16.100.42:443 weight=1 max_fails=2 fail_timeout=2;
server 172.16.100.44:443 weight=1 max_fails=2 fail_timeout=2;
}
server {
listen 8080;
server_name localhost;
root /web/errorpages;
index index.html;
}
server {
listen 80;
server_name localhost;
location / {
proxy_pass http://http-shop/;
proxy_set_header X-RIP $remote_addr;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root html;
}
}
server {
listen 443 ssl;
server_name localhost;
ssl_certificate /etc/nginx/server.crt;
ssl_certificate_key /etc/nginx/server.key;
location / {
proxy_pass https://https-shop/;
proxy_set_header X-RIP $remote_addr;
}
}
}提供sorry页面
[root@ha1 ~]# mkdir /web/errorpages -pv mkdir: created directory `/web' mkdir: created directory `/web/errorpages' [root@ha1 ~]# echo "sorry..." >/web/errorpages/index.html
拷贝https需要的证书
[root@web3 ~]# scp /etc/httpd/server.* 172.16.100.71:/etc/nginx/ root@172.16.100.71's password: server.crt 100% 3841 3.8KB/s 00:00 server.key 100% 887 0.9KB/s 00:00 [root@ha1 ~]# nginx -t nginx: the configuration file /etc/nginx/nginx.conf syntax is ok nginx: [emerg] mkdir() "/var/tmp/nginx/client/" failed (2: No such file or directory) nginx: configuration file /etc/nginx/nginx.conf test failed [root@ha1 ~]# mkdir /var/tmp/nginx [root@ha1 ~]# chown nginx !$ chown nginx /var/tmp/nginx [root@ha1 ~]# nginx -t nginx: the configuration file /etc/nginx/nginx.conf syntax is ok nginx: configuration file /etc/nginx/nginx.conf test is successful [root@ha1 ~]# service nginx start Starting nginx: [ OK ]
测试：
[root@web3 ~]# tail /usr/local/apache/logs/access_log 172.16.100.71 - - [02/Sep/2013:23:55:25 +0800] "GET /p_w_picpaths/200905/thumb_img/20_thumb_G_1242106490058.jpg HTTP/1.0" 200 1889 172.16.100.71 - - [02/Sep/2013:23:55:25 +0800] "GET /p_w_picpaths/200905/thumb_img/23_thumb_G_1241971556399.jpg HTTP/1.0" 200 2452 172.16.100.71 - - [02/Sep/2013:23:55:25 +0800] "GET /p_w_picpaths/200905/thumb_img/13_thumb_G_1241968002527.jpg HTTP/1.0" 200 2172 172.16.100.71 - - [02/Sep/2013:23:55:25 +0800] "GET /p_w_picpaths/200905/thumb_img/14_thumb_G_1241968492116.jpg HTTP/1.0" 200 2186 172.16.100.71 - - [02/Sep/2013:23:55:25 +0800] "GET /themes/default/p_w_picpaths/bnt_home.gif HTTP/1.0" 200 562 172.16.100.71 - - [02/Sep/2013:23:55:25 +0800] "GET /themes/default/p_w_picpaths/footerLine.gif HTTP/1.0" 200 48 172.16.100.71 - - [02/Sep/2013:23:55:25 +0800] "GET /themes/default/p_w_picpaths/xml_rss2.gif HTTP/1.0" 200 605 172.16.100.71 - - [02/Sep/2013:23:55:28 +0800] "GET /data/afficheimg/20081027angsif.jpg HTTP/1.0" 200 18832 172.16.100.71 - - [02/Sep/2013:23:55:28 +0800] "GET /data/afficheimg/20081027xuorxj.jpg HTTP/1.0" 200 20507 172.16.100.71 - - [03/Sep/2013:00:00:56 +0800] "GET / HTTP/1.0" 200 35722
后端webserver上修改日志格式：
[root@web3 ~]# vim /etc/httpd/httpd.conf
# LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
LogFormat "%{X-RIP}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined
# LogFormat "%h %l %u %t \"%r\" %>s %b" common
LogFormat "%{X-RIP}i %l %u %t \"%r\" %>s %b" common
[root@web3 ~]# service httpd restart
Stopping httpd: [ OK ]
Starting httpd: [ OK ]再次访问，查看日志：
[root@web3 ~]# tail /usr/local/apache/logs/access_log 172.16.0.1 - - [03/Sep/2013:00:18:24 +0800] "GET /themes/default/p_w_picpaths/moreBrands.gif HTTP/1.0" 304 - 172.16.0.1 - - [03/Sep/2013:00:18:24 +0800] "GET /data/brandlogo/1240803526904622792.gif HTTP/1.0" 304 - 172.16.0.1 - - [03/Sep/2013:00:18:25 +0800] "GET /p_w_picpaths/200905/thumb_img/1_thumb_G_1240902890710.jpg HTTP/1.0" 304 - 172.16.0.1 - - [03/Sep/2013:00:18:25 +0800] "GET /themes/default/p_w_picpaths/top_6.gif HTTP/1.0" 304 - 172.16.0.1 - - [03/Sep/2013:00:18:25 +0800] "GET /themes/default/p_w_picpaths/bnt_top.gif HTTP/1.0" 304 - 172.16.0.1 - - [03/Sep/2013:00:18:25 +0800] "GET /p_w_picpaths/200905/thumb_img/19_thumb_G_1241970175208.jpg HTTP/1.0" 304 - 172.16.0.1 - - [03/Sep/2013:00:18:25 +0800] "GET /p_w_picpaths/200905/thumb_img/22_thumb_G_1241971076803.jpg HTTP/1.0" 304 - 172.16.0.1 - - [03/Sep/2013:00:18:25 +0800] "GET /p_w_picpaths/200905/thumb_img/27_thumb_G_1241972894068.jpg HTTP/1.0" 304 - 172.16.0.1 - - [03/Sep/2013:00:18:25 +0800] "GET /themes/default/p_w_picpaths/top_7.gif HTTP/1.0" 304 - 172.16.0.1 - - [03/Sep/2013:00:18:25 +0800] "GET /api/cron.php?t=1378109879 HTTP/1.0" 200 -
web3停止httpd服务
[root@web3 ~]# service httpd stop Stopping httpd: [ OK ]
仍能访问：
web4也停掉httpd服务
[root@web4 ~]# service httpd stop Stopping httpd: [ OK ]
再次访问已切换到sorry server：
https测试：
ha2上配置：
[root@ha1 ~]# scp /etc/init.d/nginx 172.16.100.72:/etc/init.d/nginx [root@ha1 ~]# scp /etc/nginx/nginx.conf 172.16.100.72:/etc/nginx/nginx.conf [root@web3 ~]# scp /etc/httpd/server.* 172.16.100.72:/etc/nginx/ [root@ha2 ~]# chkconfig --add nginx [root@ha2 ~]# chkconfig nginx on [root@ha2 ~]# service nginx start Starting nginx: [ OK ]
测试：
安装keepalived实现nginx的高可用;
[root@ha1 keepalived-1.2.8]# yum -y install popt-devel
[root@ha1 ha]# tar xvf keepalived-1.2.8.tar.gz
[root@ha1 ha]# cd keepalived-1.2.8
[root@ha1 keepalived-1.2.8]# ./configure --prefix=/usr/local/keepalived
[root@ha1 keepalived-1.2.8]# make && make install
[root@ha1 ~]# cp /usr/local/keepalived/etc/rc.d/init.d/keepalived /etc/init.d/
[root@ha1 ~]# chkconfig --add keepalived
[root@ha1 ~]# chkconfig keepalived on
[root@ha1 ~]# cp /usr/local/keepalived/etc/sysconfig/keepalived /etc/sysconfig/keepalived
[root@ha1 ~]# mkdir /etc/keepalived
[root@ha1 ~]# ln -sv /usr/local/keepalived/sbin/keepalived /sbin/keepalived
`/sbin/keepalived' -> `/usr/local/keepalived/sbin/keepalived'
[root@ha1 keepalived]# vim /etc/keepalived/keepalived.conf
[root@ha1 keepalived]# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
root@sanyu.com
}
notification_email_from kanotify@sanyu.com
smtp_connect_timeout 3
smtp_server 127.0.0.1
router_id LVS_DEVEL
}
vrrp_script chk_nginx {
script "killall -0 nginx"
interval 1
weight 2
}
vrrp_script chk_mantaince_down {
script "[[ -f /etc/keepalived/down ]] && exit 1 || exit 0"
interval 1
weight 2
}
vrrp_instance VI_1 {
interface eth0
state MASTER # BACKUP for slave routers
priority 101 # 100 for BACKUP
virtual_router_id 70
garp_master_delay 1
authentication {
auth_type PASS
auth_pass pwsanyu
}
track_interface {
eth0
}
virtual_ipaddress {
172.16.100.70/16 dev eth0 label eth0:0
}
track_script {
chk_nginx
chk_mantaince_down
}
}
[root@ha1 keepalived]# scp keepalived.conf 172.16.100.72:/usr/local/keepalived/etc/keepalived/keepalived.conf
[root@ha2 keepalived]# vim keepalived.conf
state BACKUP # BACKUP for slave routers
priority 100 # 100 for BACKUP测试
[root@ha1 ~]# service keepalived start Starting keepalived: [ OK ] [root@ha2 ~]# service keepalived start Starting keepalived: [ OK ]
虚拟ip在ha1(MASTER)上
[root@ha1 ~]# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UNKNOWN qlen 1000
link/ether 00:0c:29:48:4c:f3 brd ff:ff:ff:ff:ff:ff
inet 172.16.100.71/16 brd 172.16.255.255 scope global eth0
inet 172.16.100.70/16 scope global secondary eth0:0
inet6 fe80::20c:29ff:fe48:4cf3/64 scope link
valid_lft forever preferred_lft forever
停掉ha1上的nginx服务：
[root@ha1 ~]# service nginx stop
Stopping nginx: [ OK ]
[root@ha1 ~]# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UNKNOWN qlen 1000
link/ether 00:0c:29:48:4c:f3 brd ff:ff:ff:ff:ff:ff
inet 172.16.100.71/16 brd 172.16.255.255 scope global eth0
inet6 fe80::20c:29ff:fe48:4cf3/64 scope link
valid_lft forever preferred_lft forevervip漂到了ha2上：
[root@ha2 ~]# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UNKNOWN qlen 1000
link/ether 00:0c:29:92:d3:4a brd ff:ff:ff:ff:ff:ff
inet 172.16.100.72/16 brd 172.16.255.255 scope global eth0
inet 172.16.100.70/16 scope global secondary eth0:0
inet6 fe80::20c:29ff:fe92:d34a/64 scope link
valid_lft forever preferred_lft forever
日志：
[root@ha1 ~]# tail -f /var/log/messages
Sep 14 05:44:21 localhost Keepalived_vrrp[7959]: VRRP_Script(chk_nginx) failed
Sep 14 05:44:23 localhost Keepalived_vrrp[7959]: VRRP_Instance(VI_1) Received higher prio advert
Sep 14 05:44:23 localhost Keepalived_vrrp[7959]: VRRP_Instance(VI_1) Entering BACKUP STATE
Sep 14 05:44:23 localhost Keepalived_vrrp[7959]: VRRP_Instance(VI_1) removing protocol VIPs.
Sep 14 05:44:23 localhost Keepalived_healthcheckers[7958]: Netlink reflector reports IP 172.16.100.70 removed
启动ha1上nginx后vip再次切回：
[root@ha1 ~]# service nginx start
Starting nginx: [ OK ]
[root@ha1 ~]# ip addr
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 16436 qdisc noqueue state UNKNOWN
link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
inet 127.0.0.1/8 scope host lo
inet6 ::1/128 scope host
valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UNKNOWN qlen 1000
link/ether 00:0c:29:48:4c:f3 brd ff:ff:ff:ff:ff:ff
inet 172.16.100.71/16 brd 172.16.255.255 scope global eth0
inet 172.16.100.70/16 scope global secondary eth0:0
inet6 fe80::20c:29ff:fe48:4cf3/64 scope link
valid_lft forever preferred_lft forever
[root@ha1 ~]# tail -f /var/log/messages
Sep 14 05:45:39 localhost Keepalived_vrrp[7959]: VRRP_Script(chk_nginx) succeeded
Sep 14 05:45:41 localhost Keepalived_vrrp[7959]: VRRP_Instance(VI_1) forcing a new MASTER election
Sep 14 05:45:42 localhost Keepalived_vrrp[7959]: VRRP_Instance(VI_1) Transition to MASTER STATE
Sep 14 05:45:43 localhost Keepalived_vrrp[7959]: VRRP_Instance(VI_1) Entering MASTER STATE
Sep 14 05:45:43 localhost Keepalived_vrrp[7959]: VRRP_Instance(VI_1) setting protocol VIPs.
Sep 14 05:45:43 localhost Keepalived_vrrp[7959]: VRRP_Instance(VI_1) Sending gratuitous ARPs on eth0 for 172.16.100.70
Sep 14 05:45:43 localhost Keepalived_healthcheckers[7958]: Netlink reflector reports IP 172.16.100.70 added
Sep 14 05:45:44 localhost Keepalived_vrrp[7959]: VRRP_Instance(VI_1) Sending gratuitous ARPs on eth0 for 172.16.100.70
