DRBD+Heartbeat+MySQL高可用

Read

基础环境：

[root@DB1 ~]# uname -a
Linux DB1.MySQL.hz.t4x.org 2.6.32-642.4.2.el6.x86_64 #1 SMP Tue Aug 23 19:58:13 UTC 2016 x86_64 x86_64 x86_64 GNU/Linux
[root@DB1 ~]# uname -n
DB1.mysql.hz.t4x.org
[root@DB1 ~]# uname -r
2.6.32-642.4.2.el6.x86_64
[root@DB1 ~]# cat /etc/redhat-release
CentOS release 6.8 (Final) SourceByrd's Weblog-https://note.t4x.org/service/drbd-heartbeat-mysql/

IP地址规划：SourceByrd's Weblog-https://note.t4x.org/service/drbd-heartbeat-mysql/

[root@DB1 ~]# ifconfig | grep "inet addr:"
inet addr:1.1.1.110 Bcast:1.1.1.255 Mask:255.255.255.0 eth0
inet addr:10.162.32.110 Bcast:10.162.47.255 Mask:255.255.240.0 eth1
inet addr:172.16.0.110 Bcast:172.16.255.255 Mask:255.255.0.0 eht2SourceByrd's Weblog-https://note.t4x.org/service/drbd-heartbeat-mysql/
[root@DB2 ~]# ifconfig | grep "inet addr:"
inet addr:1.1.1.111 Bcast:1.1.1.255 Mask:255.255.255.0 eth0
inet addr:10.162.32.111 Bcast:10.162.47.255 Mask:255.255.240.0 eth1
inet addr:172.16.0.111 Bcast:172.16.255.255 Mask:255.255.0.0 eht2 SourceByrd's Weblog-https://note.t4x.org/service/drbd-heartbeat-mysql/

必要的配置：

[root@DB1 ~]# tail -2 /etc/hosts
1.1.1.110    DB1.mysql.hz.t4x.org
1.1.1.111    DB2.mysql.hz.t4x.org
[root@DB1 ~]# tail -4 /etc/rc.local
#heartbeat
route add -host 1.1.1.111 dev eth1
#drbd
route add -host 172.16.0.111 dev eth2

[root@DB1 ~]# tail -2 /etc/hosts

1.1.1.110 DB1.mysql.hz.t4x.org

1.1.1.111 DB2.mysql.hz.t4x.org

[root@DB1 ~]# tail -4 /etc/rc.local

#heartbeat

route add -host 1.1.1.111 dev eth1

#drbd

route add -host 172.16.0.111 dev eth2

mysql配置：

[root@DB1 tools]# yum install vim gcc gcc-c++ wget tree lsof ncurses-devel cmake libaio* -y
[root@DB1 tools]# tar zxf mariadb-10.1.13.tar.gz
[root@DB1 tools]# cd mariadb-10.1.13
[root@DB1 mariadb-10.1.13]# groupadd -r mysql -g 27
[root@DB1 mariadb-10.1.13]# useradd -r mysql -u 27 -g mysql -d /data/ -s /sbin/nologin
[root@DB1 mariadb-10.1.13]# cmake . -DCMAKE_INSTALL_PREFIX=/byrd/service/mariadb-10.1.13 -DDEFAULT_CHARSET=utf8 -DDEFAULT_COLLATION=utf8_general_ci -DENABLED_LOCAL_INFILE=ON -DWITH_READLINE=1 -DWITH_PARTITION_STORAGE_ENGINE=1 -DWITH_MYISAM_STORAGE_ENGINE=1 -DWITH_INNOBASE_STORAGE_ENGINE=1 -DWITH_FEDERATED_STORAGE_ENGINE=1 -DWITH_BLACKHOLE_STORAGE_ENGINE=1 -DWITHOUT_EXAMPLE_STORAGE_ENGINE=1 -DWITH_MEMORY_STORAGE_ENGINE=1
[root@DB1 mariadb-10.1.13]# make && make install
[root@DB1 mariadb-10.1.13]# ln -s /byrd/service/mariadb-10.1.13 /usr/local/mysql
[root@DB1 mariadb-10.1.13]# /usr/local/mysql/scripts/mysql_install_db --basedir=/usr/local/mysql/ --datadir=/data/3306/data --user=mysql
[root@DB1 mariadb-10.1.13]# /usr/local/mysql/scripts/mysql_install_db --basedir=/usr/local/mysql/ --datadir=/data/3307/data --user=mysql
[root@DB1 tools]# chown -R mysql:mysql /data/
[root@DB1 mariadb-10.1.13]# cd /data/3306/
[root@DB1 3306]# mkdir {scripts,log,etc}
[root@DB1 3306]# chown -R mysql.mysql log
[root@DB1 3306]# chown -R mysql.mysql /usr/local/mysql/data/    #手动测试的时候pid在data目录下导致无法启动mysql

[root@DB1 ~]# cp /data/3306/scripts/mysqld /etc/ha.d/resource.d/
[root@DB1 ~]# chmod +x /etc/ha.d/resource.d/mysqld  
[root@DB1 ~]# scp -P22 /data/3306/scripts/mysqld root@1.1.1.111:/etc/ha.d/resource.d/
[root@DB2 /]# chmod +x /etc/ha.d/resource.d/mysqld

[root@DB1 tools]# yum install vim gcc gcc-c++ wget tree lsof ncurses-devel cmake libaio* -y

[root@DB1 tools]# tar zxf mariadb-10.1.13.tar.gz

[root@DB1 tools]# cd mariadb-10.1.13

[root@DB1 mariadb-10.1.13]# groupadd -r mysql -g 27

[root@DB1 mariadb-10.1.13]# useradd -r mysql -u 27 -g mysql -d /data/ -s /sbin/nologin

[root@DB1 mariadb-10.1.13]# cmake . -DCMAKE_INSTALL_PREFIX=/byrd/service/mariadb-10.1.13 -DDEFAULT_CHARSET=utf8 -DDEFAULT_COLLATION=utf8_general_ci -DENABLED_LOCAL_INFILE=ON -DWITH_READLINE=1 -DWITH_PARTITION_STORAGE_ENGINE=1 -DWITH_MYISAM_STORAGE_ENGINE=1 -DWITH_INNOBASE_STORAGE_ENGINE=1 -DWITH_FEDERATED_STORAGE_ENGINE=1 -DWITH_BLACKHOLE_STORAGE_ENGINE=1 -DWITHOUT_EXAMPLE_STORAGE_ENGINE=1 -DWITH_MEMORY_STORAGE_ENGINE=1

[root@DB1 mariadb-10.1.13]# make && make install

[root@DB1 mariadb-10.1.13]# ln -s /byrd/service/mariadb-10.1.13 /usr/local/mysql

[root@DB1 mariadb-10.1.13]# /usr/local/mysql/scripts/mysql_install_db --basedir=/usr/local/mysql/ --datadir=/data/3306/data --user=mysql

[root@DB1 mariadb-10.1.13]# /usr/local/mysql/scripts/mysql_install_db --basedir=/usr/local/mysql/ --datadir=/data/3307/data --user=mysql

[root@DB1 tools]# chown -R mysql:mysql /data/

[root@DB1 mariadb-10.1.13]# cd /data/3306/

[root@DB1 3306]# mkdir {scripts,log,etc}

[root@DB1 3306]# chown -R mysql.mysql log

[root@DB1 3306]# chown -R mysql.mysql /usr/local/mysql/data/ #手动测试的时候pid在data目录下导致无法启动mysql

[root@DB1 ~]# cp /data/3306/scripts/mysqld /etc/ha.d/resource.d/

[root@DB1 ~]# chmod +x /etc/ha.d/resource.d/mysqld

[root@DB1 ~]# scp -P22 /data/3306/scripts/mysqld root@1.1.1.111:/etc/ha.d/resource.d/

[root@DB2 /]# chmod +x /etc/ha.d/resource.d/mysqld

速度测试：

[root@DB1 ~]# hdparm -tT --direct /dev/drbd0 

/dev/drbd0:
 Timing O_DIRECT cached reads:   398 MB in  2.00 seconds = 199.00 MB/sec
 Timing O_DIRECT disk reads: 280 MB in  3.01 seconds =  93.05 MB/sec

[root@DB1 ~]# hdparm -tT --direct /dev/drbd0

/dev/drbd0:

Timing O_DIRECT cached reads: 398 MB in 2.00 seconds = 199.00 MB/sec

Timing O_DIRECT disk reads: 280 MB in 3.01 seconds = 93.05 MB/sec

haresources文件：【参考heartbeat】

[root@DB1 ~]# cat /etc/ha.d/haresources 
# Heartberat service
#DB1.mysql.hz.t4x.org IPaddr::1.1.1.200/24/eth0
DB1.mysql.hz.t4x.org IPaddr::1.1.1.200/24/eth0 drbddisk::data Filesystem::/dev/drbd0::/data::ext4 mysqld

[root@DB1 ~]# cat /etc/ha.d/haresources

# Heartberat service

#DB1.mysql.hz.t4x.org IPaddr::1.1.1.200/24/eth0

DB1.mysql.hz.t4x.org IPaddr::1.1.1.200/24/eth0 drbddisk::data Filesystem::/dev/drbd0::/data::ext4 mysqld

MY.CNF配置文件：

[root@DB1 3306]# cat etc/my.cnf
[client]
port    = 3306
socket  = /data/3306/log/mysql.sock
[mysqld]
user    = mysql
port    = 3306
socket  = /data/3306/log/mysql.sock
basedir = /usr/local/mysql
datadir = /data/3306/data
pid-file =  /usr/local/mysql/data/master_mysql.pid
log-bin = /data/3306/mysql-bin
slave-skip-errors = 1032,1062
replicate-ignore-db=mysql
server-id = 1
[mysqld_safe]
log-error= /data/3306/log/mysql_3306.err
pid-file= /usr/local/mysql/data/master_mysql.pid

[root@DB1 3306]# cat etc/my.cnf

[client]

port = 3306

socket = /data/3306/log/mysql.sock

[mysqld]

user = mysql

port = 3306

socket = /data/3306/log/mysql.sock

basedir = /usr/local/mysql

datadir = /data/3306/data

pid-file = /usr/local/mysql/data/master_mysql.pid

log-bin = /data/3306/mysql-bin

slave-skip-errors = 1032,1062

replicate-ignore-db=mysql

server-id = 1

[mysqld_safe]

log-error= /data/3306/log/mysql_3306.err

pid-file= /usr/local/mysql/data/master_mysql.pid

日志分析(DB1启动heartbeat日志 )：

Sep 07 17:31:18 DB1.mysql.hz.t4x.org heartbeat: [8292]: info: Pacemaker support: no
Sep 07 17:31:18 DB1.mysql.hz.t4x.org heartbeat: [8292]: WARN: Logging daemon is disabled --enabling logging daemon is recommended
Sep 07 17:31:18 DB1.mysql.hz.t4x.org heartbeat: [8292]: info: **************************
Sep 07 17:31:18 DB1.mysql.hz.t4x.org heartbeat: [8292]: info: Configuration validated. Starting heartbeat 3.0.6
Sep 07 17:31:18 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: heartbeat: version 3.0.6
Sep 07 17:31:18 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: Heartbeat generation: 1473172449
Sep 07 17:31:18 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: glib: UDP multicast heartbeat started for group 225.0.0.18 port 694 interface eth1 (ttl=1 loop=0)
Sep 07 17:31:19 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: Local status now set to: 'up'
Sep 07 17:31:20 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: Link db2.mysql.hz.t4x.org:eth1 up.
Sep 07 17:31:20 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: Status update for node db2.mysql.hz.t4x.org: status active
harc(default)[8302]:    2016/09/07_17:31:20 info: Running /etc/ha.d//rc.d/status status
Sep 07 17:31:20 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: Comm_now_up(): updating status to active
Sep 07 17:31:20 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: Local status now set to: 'active'
Sep 07 17:31:21 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: remote resource transition completed.
Sep 07 17:31:21 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: remote resource transition completed.
Sep 07 17:31:21 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: Local Resource acquisition completed. (none)
Sep 07 17:31:21 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: db2.mysql.hz.t4x.org wants to go standby [foreign]
Sep 07 17:31:27 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: standby: acquire [foreign] resources from db2.mysql.hz.t4x.org
Sep 07 17:31:27 DB1.mysql.hz.t4x.org heartbeat: [8322]: info: acquire local HA resources (standby).
ResourceManager(default)[8335]: 2016/09/07_17:31:27 info: Acquiring resource group: db1.mysql.hz.t4x.org IPaddr::1.1.1.200/24/eth0 drbddisk::data Filesystem::/dev/drbd0::/data::ext4 mysqld
/usr/lib/ocf/resource.d//heartbeat/IPaddr(IPaddr_1.1.1.200)[8363]:      2016/09/07_17:31:27 INFO:  Resource is stopped
ResourceManager(default)[8335]: 2016/09/07_17:31:27 info: Running /etc/ha.d/resource.d/IPaddr 1.1.1.200/24/eth0 start
IPaddr(IPaddr_1.1.1.200)[8448]: 2016/09/07_17:31:27 INFO: Using calculated netmask for 1.1.1.200: 255.255.255.0
IPaddr(IPaddr_1.1.1.200)[8448]: 2016/09/07_17:31:28 INFO: eval ifconfig eth0:4 1.1.1.200 netmask 255.255.255.0 broadcast 1.1.1.255
/usr/lib/ocf/resource.d//heartbeat/IPaddr(IPaddr_1.1.1.200)[8422]:      2016/09/07_17:31:28 INFO:  Success
ResourceManager(default)[8335]: 2016/09/07_17:31:28 info: Running /etc/ha.d/resource.d/drbddisk data start
/usr/lib/ocf/resource.d//heartbeat/Filesystem(Filesystem_/dev/drbd0)[8587]:     2016/09/07_17:31:29 INFO:  Resource is stopped
ResourceManager(default)[8335]: 2016/09/07_17:31:29 info: Running /etc/ha.d/resource.d/Filesystem /dev/drbd0 /data ext4 start
Filesystem(Filesystem_/dev/drbd0)[8676]:        2016/09/07_17:31:29 INFO: Running start for /dev/drbd0 on /data
/usr/lib/ocf/resource.d//heartbeat/Filesystem(Filesystem_/dev/drbd0)[8668]:     2016/09/07_17:31:30 INFO:  Success
ResourceManager(default)[8335]: 2016/09/07_17:31:31 info: Running /etc/ha.d/resource.d/mysqld  start
Sep 07 17:31:35 DB1.mysql.hz.t4x.org heartbeat: [8322]: info: local HA resource acquisition completed (standby).
Sep 07 17:31:35 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: Standby resource acquisition done [foreign].
Sep 07 17:31:35 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: Initial resource acquisition complete (auto_failback)
Sep 07 17:31:35 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: remote resource transition completed.

Sep 07 17:31:18 DB1.mysql.hz.t4x.org heartbeat: [8292]: info: Pacemaker support: no

Sep 07 17:31:18 DB1.mysql.hz.t4x.org heartbeat: [8292]: WARN: Logging daemon is disabled --enabling logging daemon is recommended

Sep 07 17:31:18 DB1.mysql.hz.t4x.org heartbeat: [8292]: info: **************************

Sep 07 17:31:18 DB1.mysql.hz.t4x.org heartbeat: [8292]: info: Configuration validated. Starting heartbeat 3.0.6

Sep 07 17:31:18 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: heartbeat: version 3.0.6

Sep 07 17:31:18 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: Heartbeat generation: 1473172449

Sep 07 17:31:18 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: glib: UDP multicast heartbeat started for group 225.0.0.18 port 694 interface eth1 (ttl=1 loop=0)

Sep 07 17:31:19 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: Local status now set to: 'up'

Sep 07 17:31:20 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: Link db2.mysql.hz.t4x.org:eth1 up.

Sep 07 17:31:20 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: Status update for node db2.mysql.hz.t4x.org: status active

harc(default)[8302]: 2016/09/07_17:31:20 info: Running /etc/ha.d//rc.d/status status

Sep 07 17:31:20 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: Comm_now_up(): updating status to active

Sep 07 17:31:20 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: Local status now set to: 'active'

Sep 07 17:31:21 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: remote resource transition completed.

Sep 07 17:31:21 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: Local Resource acquisition completed. (none)

Sep 07 17:31:21 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: db2.mysql.hz.t4x.org wants to go standby [foreign]

Sep 07 17:31:27 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: standby: acquire [foreign] resources from db2.mysql.hz.t4x.org

Sep 07 17:31:27 DB1.mysql.hz.t4x.org heartbeat: [8322]: info: acquire local HA resources (standby).

ResourceManager(default)[8335]: 2016/09/07_17:31:27 info: Acquiring resource group: db1.mysql.hz.t4x.org IPaddr::1.1.1.200/24/eth0 drbddisk::data Filesystem::/dev/drbd0::/data::ext4 mysqld

/usr/lib/ocf/resource.d//heartbeat/IPaddr(IPaddr_1.1.1.200)[8363]: 2016/09/07_17:31:27 INFO: Resource is stopped

ResourceManager(default)[8335]: 2016/09/07_17:31:27 info: Running /etc/ha.d/resource.d/IPaddr 1.1.1.200/24/eth0 start

IPaddr(IPaddr_1.1.1.200)[8448]: 2016/09/07_17:31:27 INFO: Using calculated netmask for 1.1.1.200: 255.255.255.0

IPaddr(IPaddr_1.1.1.200)[8448]: 2016/09/07_17:31:28 INFO: eval ifconfig eth0:4 1.1.1.200 netmask 255.255.255.0 broadcast 1.1.1.255

/usr/lib/ocf/resource.d//heartbeat/IPaddr(IPaddr_1.1.1.200)[8422]: 2016/09/07_17:31:28 INFO: Success

ResourceManager(default)[8335]: 2016/09/07_17:31:28 info: Running /etc/ha.d/resource.d/drbddisk data start

/usr/lib/ocf/resource.d//heartbeat/Filesystem(Filesystem_/dev/drbd0)[8587]: 2016/09/07_17:31:29 INFO: Resource is stopped

ResourceManager(default)[8335]: 2016/09/07_17:31:29 info: Running /etc/ha.d/resource.d/Filesystem /dev/drbd0 /data ext4 start

Filesystem(Filesystem_/dev/drbd0)[8676]: 2016/09/07_17:31:29 INFO: Running start for /dev/drbd0 on /data

/usr/lib/ocf/resource.d//heartbeat/Filesystem(Filesystem_/dev/drbd0)[8668]: 2016/09/07_17:31:30 INFO: Success

ResourceManager(default)[8335]: 2016/09/07_17:31:31 info: Running /etc/ha.d/resource.d/mysqld start

Sep 07 17:31:35 DB1.mysql.hz.t4x.org heartbeat: [8322]: info: local HA resource acquisition completed (standby).

Sep 07 17:31:35 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: Standby resource acquisition done [foreign].

Sep 07 17:31:35 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: Initial resource acquisition complete (auto_failback)

Sep 07 17:31:35 DB1.mysql.hz.t4x.org heartbeat: [8293]: info: remote resource transition completed.

日志分析(DB1关机 DB2日志)：

Sep 07 18:12:50 DB2.mysql.hz.t4x.org heartbeat: [4367]: WARN: Late heartbeat: Node db1.mysql.hz.t4x.org: interval 16040 ms
Sep 07 18:13:51 DB2.mysql.hz.t4x.org heartbeat: [4367]: WARN: node db1.mysql.hz.t4x.org: is dead
Sep 07 18:13:51 DB2.mysql.hz.t4x.org heartbeat: [4367]: WARN: No STONITH device configured.
Sep 07 18:13:51 DB2.mysql.hz.t4x.org heartbeat: [4367]: WARN: Shared disks are not protected.
Sep 07 18:13:51 DB2.mysql.hz.t4x.org heartbeat: [4367]: info: Resources being acquired from db1.mysql.hz.t4x.org.
Sep 07 18:13:51 DB2.mysql.hz.t4x.org heartbeat: [4367]: info: Link db1.mysql.hz.t4x.org:eth1 dead.
harc(default)[8150]:    2016/09/07_18:13:52 info: Running /etc/ha.d//rc.d/status status
Sep 07 18:13:52 DB2.mysql.hz.t4x.org heartbeat: [8151]: info: No local resources [/byrd/service/heartbeat/share/heartbeat/ResourceManager listkeys db2.mysql.hz.t4x.org] to acquire.
mach_down(default)[8180]:       2016/09/07_18:13:52 info: Taking over resource group IPaddr::1.1.1.200/24/eth0
ResourceManager(default)[8207]: 2016/09/07_18:13:52 info: Acquiring resource group: db1.mysql.hz.t4x.org IPaddr::1.1.1.200/24/eth0 drbddisk::data Filesystem::/dev/drbd0::/data::ext4 mysqld
/usr/lib/ocf/resource.d//heartbeat/IPaddr(IPaddr_1.1.1.200)[8235]:      2016/09/07_18:13:52 INFO:  Resource is stopped
ResourceManager(default)[8207]: 2016/09/07_18:13:52 info: Running /etc/ha.d/resource.d/IPaddr 1.1.1.200/24/eth0 start
IPaddr(IPaddr_1.1.1.200)[8320]: 2016/09/07_18:13:53 INFO: Using calculated netmask for 1.1.1.200: 255.255.255.0
IPaddr(IPaddr_1.1.1.200)[8320]: 2016/09/07_18:13:53 INFO: eval ifconfig eth0:1 1.1.1.200 netmask 255.255.255.0 broadcast 1.1.1.255
/usr/lib/ocf/resource.d//heartbeat/IPaddr(IPaddr_1.1.1.200)[8294]:      2016/09/07_18:13:53 INFO:  Success
ResourceManager(default)[8207]: 2016/09/07_18:13:53 info: Running /etc/ha.d/resource.d/drbddisk data start
/usr/lib/ocf/resource.d//heartbeat/Filesystem(Filesystem_/dev/drbd0)[8457]:     2016/09/07_18:13:54 INFO:  Resource is stopped
ResourceManager(default)[8207]: 2016/09/07_18:13:54 info: Running /etc/ha.d/resource.d/Filesystem /dev/drbd0 /data ext4 start
Filesystem(Filesystem_/dev/drbd0)[8547]:        2016/09/07_18:13:55 INFO: Running start for /dev/drbd0 on /data
/usr/lib/ocf/resource.d//heartbeat/Filesystem(Filesystem_/dev/drbd0)[8539]:     2016/09/07_18:13:56 INFO:  Success
ResourceManager(default)[8207]: 2016/09/07_18:13:56 info: Running /etc/ha.d/resource.d/mysqld  start
mach_down(default)[8180]:       2016/09/07_18:14:02 info: /byrd/service/heartbeat/share/heartbeat/mach_down: nice_failback: foreign resources acquired
mach_down(default)[8180]:       2016/09/07_18:14:02 info: mach_down takeover complete for node db1.mysql.hz.t4x.org.
Sep 07 18:14:02 DB2.mysql.hz.t4x.org heartbeat: [4367]: info: mach_down takeover complete.

Sep 07 18:12:50 DB2.mysql.hz.t4x.org heartbeat: [4367]: WARN: Late heartbeat: Node db1.mysql.hz.t4x.org: interval 16040 ms

Sep 07 18:13:51 DB2.mysql.hz.t4x.org heartbeat: [4367]: WARN: node db1.mysql.hz.t4x.org: is dead

Sep 07 18:13:51 DB2.mysql.hz.t4x.org heartbeat: [4367]: WARN: No STONITH device configured.

Sep 07 18:13:51 DB2.mysql.hz.t4x.org heartbeat: [4367]: WARN: Shared disks are not protected.

Sep 07 18:13:51 DB2.mysql.hz.t4x.org heartbeat: [4367]: info: Resources being acquired from db1.mysql.hz.t4x.org.

Sep 07 18:13:51 DB2.mysql.hz.t4x.org heartbeat: [4367]: info: Link db1.mysql.hz.t4x.org:eth1 dead.

harc(default)[8150]: 2016/09/07_18:13:52 info: Running /etc/ha.d//rc.d/status status

Sep 07 18:13:52 DB2.mysql.hz.t4x.org heartbeat: [8151]: info: No local resources [/byrd/service/heartbeat/share/heartbeat/ResourceManager listkeys db2.mysql.hz.t4x.org] to acquire.

mach_down(default)[8180]: 2016/09/07_18:13:52 info: Taking over resource group IPaddr::1.1.1.200/24/eth0

ResourceManager(default)[8207]: 2016/09/07_18:13:52 info: Acquiring resource group: db1.mysql.hz.t4x.org IPaddr::1.1.1.200/24/eth0 drbddisk::data Filesystem::/dev/drbd0::/data::ext4 mysqld

/usr/lib/ocf/resource.d//heartbeat/IPaddr(IPaddr_1.1.1.200)[8235]: 2016/09/07_18:13:52 INFO: Resource is stopped

ResourceManager(default)[8207]: 2016/09/07_18:13:52 info: Running /etc/ha.d/resource.d/IPaddr 1.1.1.200/24/eth0 start

IPaddr(IPaddr_1.1.1.200)[8320]: 2016/09/07_18:13:53 INFO: Using calculated netmask for 1.1.1.200: 255.255.255.0

IPaddr(IPaddr_1.1.1.200)[8320]: 2016/09/07_18:13:53 INFO: eval ifconfig eth0:1 1.1.1.200 netmask 255.255.255.0 broadcast 1.1.1.255

/usr/lib/ocf/resource.d//heartbeat/IPaddr(IPaddr_1.1.1.200)[8294]: 2016/09/07_18:13:53 INFO: Success

ResourceManager(default)[8207]: 2016/09/07_18:13:53 info: Running /etc/ha.d/resource.d/drbddisk data start

/usr/lib/ocf/resource.d//heartbeat/Filesystem(Filesystem_/dev/drbd0)[8457]: 2016/09/07_18:13:54 INFO: Resource is stopped

ResourceManager(default)[8207]: 2016/09/07_18:13:54 info: Running /etc/ha.d/resource.d/Filesystem /dev/drbd0 /data ext4 start

Filesystem(Filesystem_/dev/drbd0)[8547]: 2016/09/07_18:13:55 INFO: Running start for /dev/drbd0 on /data

/usr/lib/ocf/resource.d//heartbeat/Filesystem(Filesystem_/dev/drbd0)[8539]: 2016/09/07_18:13:56 INFO: Success

ResourceManager(default)[8207]: 2016/09/07_18:13:56 info: Running /etc/ha.d/resource.d/mysqld start

mach_down(default)[8180]: 2016/09/07_18:14:02 info: /byrd/service/heartbeat/share/heartbeat/mach_down: nice_failback: foreign resources acquired

mach_down(default)[8180]: 2016/09/07_18:14:02 info: mach_down takeover complete for node db1.mysql.hz.t4x.org.

Sep 07 18:14:02 DB2.mysql.hz.t4x.org heartbeat: [4367]: info: mach_down takeover complete.

效果测试：（echo速度没有写入数据库快，大概意思应该测试完成了。）

MariaDB [(none)]> update mysql.user set password=PASSWORD('123456') where user='root'; 
MariaDB [(none)]> flush privileges;
MariaDB [(none)]> create database byrd;
MariaDB [byrd]> create table byrd(id int);
[root@DB1 ~]# cat mysql.sh 
#!/bin/bash
# Author:Byrd
# Version:0.1
# Site:note.t4x.org
# Contact:root#t4x.org
while true
do
/usr/local/mysql/bin/mysql -uroot -p'123456' -S /data/3306/log/mysql.sock -e "insert into byrd.byrd values($RANDOM);"
usleep 1000    #1秒=1000000微妙 
/usr/local/mysql/bin/mysql -uroot -p123456 -S /data/3306/log/mysql.sock -e 'select * from byrd.byrd;' | sed '1d' | wc -l >> /tmp/abc.txt
echo $(date +%F-%H-%M-%S) >> /tmp/abc.txt
done



[root@DB1 ~]# tail -4 /tmp/abc.txt 
3976
2016-09-14-23-01-28
3977
2016-0
[root@DB1 ~]# drbdadm up data
Marked additional 264 MB as out-of-sync based on AL.
[root@DB1 ~]# cat /proc/drbd 
version: 8.4.4 (api:1/proto:86-101)
GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by root@DB1.mysql.hz.t4x.org, 2016-09-06 23:15:28
 0: cs:SyncTarget ro:Secondary/Primary ds:Inconsistent/UpToDate C r-----
    ns:0 nr:149800 dw:124416 dr:0 al:0 bm:9 lo:38 pe:4 ua:36 ap:0 ep:1 wo:d oos:145928
        [========>...........] sync'ed: 47.8% (145928/270344)K
        finish: 0:00:12 speed: 11,308 (11,308) want: 0 K/sec

[root@DB1 ~]# /usr/local/mysql/bin/mysql -uroot -p123456 -S /data/3306/log/mysql.sock -e 'select * from byrd.byrd;' | sed '1d' | wc -l
4215


[root@DB2 ~]# /usr/local/mysql/bin/mysql -uroot -p123456 -S /data/3306/log/mysql.sock -e 'select * from byrd.byrd;' | sed '1d' | wc -l
4215

MariaDB [(none)]> update mysql.user set password=PASSWORD('123456') where user='root';

MariaDB [(none)]> flush privileges;

MariaDB [(none)]> create database byrd;

MariaDB [byrd]> create table byrd(id int);

[root@DB1 ~]# cat mysql.sh

#!/bin/bash

# Author:Byrd

# Version:0.1

# Site:note.t4x.org

# Contact:root#t4x.org

while true

/usr/local/mysql/bin/mysql -uroot -p'123456' -S /data/3306/log/mysql.sock -e "insert into byrd.byrd values($RANDOM);"

usleep 1000 #1秒=1000000微妙

/usr/local/mysql/bin/mysql -uroot -p123456 -S /data/3306/log/mysql.sock -e 'select * from byrd.byrd;' | sed '1d' | wc -l >> /tmp/abc.txt

echo $(date +%F-%H-%M-%S) >> /tmp/abc.txt

done

[root@DB1 ~]# tail -4 /tmp/abc.txt

3976

2016-09-14-23-01-28

3977

2016-0

[root@DB1 ~]# drbdadm up data

Marked additional 264 MB as out-of-sync based on AL.

[root@DB1 ~]# cat /proc/drbd

version: 8.4.4 (api:1/proto:86-101)

GIT-hash: 74402fecf24da8e5438171ee8c19e28627e1c98a build by root@DB1.mysql.hz.t4x.org, 2016-09-06 23:15:28

0: cs:SyncTarget ro:Secondary/Primary ds:Inconsistent/UpToDate C r-----

ns:0 nr:149800 dw:124416 dr:0 al:0 bm:9 lo:38 pe:4 ua:36 ap:0 ep:1 wo:d oos:145928

[========>...........] sync'ed: 47.8% (145928/270344)K

finish: 0:00:12 speed: 11,308 (11,308) want: 0 K/sec

[root@DB1 ~]# /usr/local/mysql/bin/mysql -uroot -p123456 -S /data/3306/log/mysql.sock -e 'select * from byrd.byrd;' | sed '1d' | wc -l

4215

[root@DB2 ~]# /usr/local/mysql/bin/mysql -uroot -p123456 -S /data/3306/log/mysql.sock -e 'select * from byrd.byrd;' | sed '1d' | wc -l

4215

启动脚本：

[root@DB1 3306]# cat scripts/mysqld 
#!/bin/bash
# Author:Byrd
# Version:0.1
# Site:note.t4x.org
# Contact:root#t4x.org
. /etc/init.d/functions
MSL_USER="root"
MSL_PASS="123456"
BIN_DIR="/usr/local/mysql/bin"
CNF_FILE="/data/3306/etc/my.cnf"
PID_FILE="/usr/local/mysql/data/master_mysql.pid"
MSL_SOCK="/data/3306/log/mysql.sock"
CHK_PROC=`netstat -tunlp|grep mysql|wc -l`
MSL_ADMN="${BIN_DIR}/mysql -u${MSL_USER} -p${MSL_PASS} -S ${MSL_SOCK}"
MSL_DUMP="${BIN_DIR}/mysqldump -u${MSL_USER} -p${MSL_PASS} -S ${MSL_SOCK}"
MSL_STAT="${BIN_DIR}/mysqld_safe --defaults-file=${CNF_FILE}" 
PATH="$mysql_Bin:/sbin:/usr/sbin:/bin:/usr/bin"
export PATH
start() {
# pidfile and sock must existence
    if [ -s ${PID_FILE} -a -S ${MSL_SOCK} ]; then
            action "MySQL is running"  /bin/true
# sock does not exist
    elif [ ! -S ${MSL_SOCK} ];then
        if [ -s ${PID_FILE} ];then
        PID=`cat ${PID_FILE}`
            if (kill -9 $PID 2>/dev/null);then   #Check for the existence of the process.pid exists or not
#               echo "Shutting down MySQL"
                kill ${PID}
                sleep 3
                ${MSL_STAT} >/dev/null 2>&1 &
                RETURN_VALUE=$?
                [ "${RETURN_VALUE}" -eq 0 ] && action "Restarting MySQL" /bin/true || action "mysql err ${RETURN_VALUE}" /bin/false
            fi
        elif [ "${CHK_PROC}" -eq 0 ]; then
            sleep 4
            ${MSL_STAT} >/dev/null 2>&1 &
            RETURN_VALUE=$?
            [ "${RETURN_VALUE}" -eq 0 ] && action "Starting MySQL" /bin/true || action "mysql err ${RETURN_VALUE}" /bin/false
        else
            #kill `ps aux|grep basedir|grep -v grep|awk '{print $2}'`
            sleep 5
            ${MSL_STAT} >/dev/null 2>&1 &
            RETURN_VALUE=$?
            [ "${RETURN_VALUE}" -eq 0 ] && action "Starting MySQL" /bin/true || action "mysql err ${RETURN_VALUE}" /bin/false
        fi
# pidfile does not exist or others
    else
        kill `ps aux|grep basedir|grep -v grep|awk '{print $2}'`
        sleep 6
        ${MSL_STAT} >/dev/null 2>&1 &
        RETURN_VALUE=$?
        [ "${RETURN_VALUE}" -eq 0 ] && action "Starting MySQL" /bin/true || action "mysql err ${RETURN_VALUE}" /bin/false
    fi
}
stop() { 
    if [ -s "${PID_FILE}" ];then
    PID=`cat ${PID_FILE}`
        if (kill -0 ${PID} 2>/dev/null);then
        kill $PID
        sleep 3 
        action "Shutting down MySQL" /bin/true
        fi
    elif [ -S ${MSL_SOCK} ]; then
        kill `ps aux|grep basedir|grep -v grep|awk '{print $2}'`
        sleep 4
        PID=`cat ${PID_FILE}`
        kill $PID
        action "Shutting down MySQL" /bin/true
    else
        action "MySQL server process #$PID is not running!" /bin/true
        rm -f $pid_File
    fi
}
case "$1" in
    start)
        start && exit 0
        $1
        ;;
    stop)
        stop || exit 2 
        ;;
    restart)
        stop
        start
        ;;
    *)
        echo $"Usage: $0 {start|stop|restart}"
        exit 2
esac
exit $?

[root@DB1 3306]# cat scripts/mysqld

#!/bin/bash

# Author:Byrd

# Version:0.1

# Site:note.t4x.org

# Contact:root#t4x.org

. /etc/init.d/functions

MSL_USER="root"

MSL_PASS="123456"

BIN_DIR="/usr/local/mysql/bin"

CNF_FILE="/data/3306/etc/my.cnf"

PID_FILE="/usr/local/mysql/data/master_mysql.pid"

MSL_SOCK="/data/3306/log/mysql.sock"

CHK_PROC=`netstat -tunlp|grep mysql|wc -l`

MSL_ADMN="${BIN_DIR}/mysql -u${MSL_USER} -p${MSL_PASS} -S ${MSL_SOCK}"

MSL_DUMP="${BIN_DIR}/mysqldump -u${MSL_USER} -p${MSL_PASS} -S ${MSL_SOCK}"

MSL_STAT="${BIN_DIR}/mysqld_safe --defaults-file=${CNF_FILE}"

PATH="$mysql_Bin:/sbin:/usr/sbin:/bin:/usr/bin"

export PATH

start() {

# pidfile and sock must existence

if [ -s ${PID_FILE} -a -S ${MSL_SOCK} ]; then

action "MySQL is running" /bin/true

# sock does not exist

elif [ ! -S ${MSL_SOCK} ];then

if [ -s ${PID_FILE} ];then

PID=`cat ${PID_FILE}`

if (kill -9 $PID 2>/dev/null);then #Check for the existence of the process.pid exists or not

# echo "Shutting down MySQL"

kill ${PID}

sleep 3

${MSL_STAT} >/dev/null 2>&1 &

RETURN_VALUE=$?

[ "${RETURN_VALUE}" -eq 0 ] && action "Restarting MySQL" /bin/true || action "mysql err ${RETURN_VALUE}" /bin/false

elif [ "${CHK_PROC}" -eq 0 ]; then

sleep 4

${MSL_STAT} >/dev/null 2>&1 &

RETURN_VALUE=$?

[ "${RETURN_VALUE}" -eq 0 ] && action "Starting MySQL" /bin/true || action "mysql err ${RETURN_VALUE}" /bin/false

else

#kill `ps aux|grep basedir|grep -v grep|awk '{print $2}'`

sleep 5

${MSL_STAT} >/dev/null 2>&1 &

RETURN_VALUE=$?

[ "${RETURN_VALUE}" -eq 0 ] && action "Starting MySQL" /bin/true || action "mysql err ${RETURN_VALUE}" /bin/false

# pidfile does not exist or others

else

kill `ps aux|grep basedir|grep -v grep|awk '{print $2}'`

sleep 6

${MSL_STAT} >/dev/null 2>&1 &

RETURN_VALUE=$?

[ "${RETURN_VALUE}" -eq 0 ] && action "Starting MySQL" /bin/true || action "mysql err ${RETURN_VALUE}" /bin/false

}

stop() {

if [ -s "${PID_FILE}" ];then

PID=`cat ${PID_FILE}`

if (kill -0 ${PID} 2>/dev/null);then

kill $PID

sleep 3

action "Shutting down MySQL" /bin/true

elif [ -S ${MSL_SOCK} ]; then

kill `ps aux|grep basedir|grep -v grep|awk '{print $2}'`

sleep 4

PID=`cat ${PID_FILE}`

kill $PID

action "Shutting down MySQL" /bin/true

else

action "MySQL server process #$PID is not running!" /bin/true

rm -f $pid_File

}

case "$1" in

start)

start && exit 0

;;

stop)

stop || exit 2

;;

restart)

stop

start

;;

echo $"Usage: $0 {start|stop|restart}"

exit 2

esac

exit $?

SourceByrd's Weblog-https://note.t4x.org/service/drbd-heartbeat-mysql/ SourceByrd's Weblog-https://note.t4x.org/service/drbd-heartbeat-mysql/

Today in History

November

申明：除非注明Byrd's Blog内容均为原创,未经许可禁止转载!详情请阅读版权申明!

Today in History

Comment

Login