VxVM发生serial split brain 的故障排除

登录系统提示有邮件，发现有hot relocation了，原来mmsdg06 磁盘失败了.

root@mmsc1 # vxdisk list
DEVICE       TYPE            DISK         GROUP        STATUS
c1t0d0s2     auto:none       -            -            online invalid
c1t2d0s2     auto:none       -            -            online invalid
c1t3d0s2     auto:sliced     -            -            online invalid
c3t8d0s2     auto:sliced     oradg01      oradg        online
c3t9d0s2     auto:sliced     mmsdg01      mmsdg        online
c3t10d0s2    auto:sliced     mmsdg02      mmsdg        online
c3t11d0s2    auto:sliced     mmsdg03      mmsdg        online
c3t12d0s2    auto:sliced     oradg03      oradg        online spare
c5t8d0s2     auto:sliced     oradg02      oradg        online
c5t9d0s2     auto:sliced     mmsdg04      mmsdg        online
c5t10d0s2    auto:sliced     mmsdg05      mmsdg        online
c5t11d0s2    auto:sliced     -            -            online
c5t12d0s2    auto:sliced     mmsdg07      mmsdg        online spare
-            -         mmsdg06      mmsdg        failed was:c5t11d0s2

尝试恢复磁盘
root@mmsc1 # vxreattach
root@mmsc1 # vxdisk list
DEVICE       TYPE            DISK         GROUP        STATUS
c1t0d0s2     auto:none       -            -            online invalid
c1t2d0s2     auto:none       -            -            online invalid
c1t3d0s2     auto:sliced     -            -            online invalid
c3t8d0s2     auto:sliced     oradg01      oradg        online
c3t9d0s2     auto:sliced     mmsdg01      mmsdg        online
c3t10d0s2    auto:sliced     mmsdg02      mmsdg        online
c3t11d0s2    auto:sliced     mmsdg03      mmsdg        online
c3t12d0s2    auto:sliced     oradg03      oradg        online spare
c5t8d0s2     auto:sliced     oradg02      oradg        online
c5t9d0s2     auto:sliced     mmsdg04      mmsdg        online
c5t10d0s2    auto:sliced     mmsdg05      mmsdg        online
c5t11d0s2    auto:sliced     mmsdg06      mmsdg        online
c5t12d0s2    auto:sliced     mmsdg07      mmsdg        online spare

reattach之后，磁盘又可以被vxvm管理了， 说明磁盘并没有物理错误。但是我还是想让mmdg07为热备盘，由于有部分数据(plex)在mmsdg07上，接下来要把mmsdg07替换成mmsdg06,就是说要把在mmsdg07数据倒换到mmsdg06上,把mmsdg07强制删除
root@mmsc1 # vxdg -g mmsdg  -k rmdisk mmsdg07
root@mmsc1 # vxdisk list
DEVICE       TYPE            DISK         GROUP        STATUS
c1t0d0s2     auto:none       -            -            online invalid
c1t2d0s2     auto:none       -            -            online invalid
c1t3d0s2     auto:sliced     -            -            online invalid
c3t8d0s2     auto:sliced     oradg01      oradg        online
c3t9d0s2     auto:sliced     mmsdg01      mmsdg        online
c3t10d0s2    auto:sliced     mmsdg02      mmsdg        online
c3t11d0s2    auto:sliced     mmsdg03      mmsdg        online
c3t12d0s2    auto:sliced     oradg03      oradg        online spare
c5t8d0s2     auto:sliced     oradg02      oradg        online
c5t9d0s2     auto:sliced     mmsdg04      mmsdg        online
c5t10d0s2    auto:sliced     mmsdg05      mmsdg        online
c5t11d0s2    auto:sliced     mmsdg06      mmsdg        online
c5t12d0s2    auto:sliced     -            -            online
-            -         mmsdg07      mmsdg        removed was:c5t12d0s2

mmsdg06替换到mmsdg07
root@mmsc1 # vxdg -g mmsdg repldisk mmsdg07=mmsdg06
root@mmsc1 # vxdisk list
DEVICE       TYPE            DISK         GROUP        STATUS
c1t0d0s2     auto:none       -            -            online invalid
c1t2d0s2     auto:none       -            -            online invalid
c1t3d0s2     auto:sliced     -            -            online invalid
c3t8d0s2     auto:sliced     oradg01      oradg        online
c3t9d0s2     auto:sliced     mmsdg01      mmsdg        online
c3t10d0s2    auto:sliced     mmsdg02      mmsdg        online
c3t11d0s2    auto:sliced     mmsdg03      mmsdg        online
c3t12d0s2    auto:sliced     oradg03      oradg        online spare
c5t8d0s2     auto:sliced     oradg02      oradg        online
c5t9d0s2     auto:sliced     mmsdg04      mmsdg        online
c5t10d0s2    auto:sliced     mmsdg05      mmsdg        online
c5t11d0s2    auto:sliced     mmsdg07      mmsdg        online
c5t12d0s2    auto:sliced     -            -            online

恢复正常名字
root@mmsc1 # vxedit -g mmsdg rename mmsdg07=mmsdg06

查询磁盘状态
root@mmsc1 # vxdisk list
DEVICE       TYPE            DISK         GROUP        STATUS
c1t0d0s2     auto:none       -            -            online invalid
c1t2d0s2     auto:none       -            -            online invalid
c1t3d0s2     auto:sliced     -            -            online invalid
c3t8d0s2     auto:sliced     oradg01      oradg        online
c3t9d0s2     auto:sliced     mmsdg01      mmsdg        online
c3t10d0s2    auto:sliced     mmsdg02      mmsdg        online
c3t11d0s2    auto:sliced     mmsdg03      mmsdg        online
c3t12d0s2    auto:sliced     oradg03      oradg        online spare
c5t8d0s2     auto:sliced     oradg02      oradg        online
c5t9d0s2     auto:sliced     mmsdg04      mmsdg        online
c5t10d0s2    auto:sliced     mmsdg05      mmsdg        online
c5t11d0s2    auto:sliced     mmsdg06      mmsdg        online
c5t12d0s2    auto:sliced     -            -            online

尝试把c5t12d0s2加回到mmsdg, 但是报错了
root@mmsc1 # vxdg -g mmsdg adddisk mmsdg07=c5t12d0s2
VxVM vxdg ERROR V-5-1-10127 associating disk-media mmsdg07 with c5t12d0s2:
        Serial Split Brain detected. Run vxsplitlines

发生了ssb，怎么办？？ ssb产生的原因一般都是系统配置不一致引起的，按提示使用vxsplitlines命令
root@mmsc1 # vxsplitlines -g  mmsdg
[ ######### ]
  VxVM vxsplitlines NOTICE V-5-2-2708 There are 1 pools.
The Following are the disks in each pool. Each disk in the same pool
has config copies that are similar.
  VxVM vxsplitlines INFO V-5-2-2707 Pool 0.
c3t9d0s2 mmsdg01
To see the configuration copy from this disk issue
/etc/vx/diag.d/vxprivutil dumpconfig /dev/vx/dmp/c3t9d0s3
To import the diskgroup with config copy from this
disk use the following command
/usr/sbin/vxdg -o selectcp=1211690416.23.mmsc1 import mmsdg
The following are the disks whose ssb ids don't match in this config
copy
mmsdg06

如果要执行上面的import话只能停止业务，没办法，先把双机停掉
root@mmsc1 # scswitch -F -g mms_rg

umount所有卷 ....
(此处省略一些操作)

把mmsdg逐出
root@mmsc1 # vxdg deport mmsdg

再使用上面的提示命令导入dg
/usr/sbin/vxdg -o selectcp=1211690416.23.mmsc1 import mmsdg

恢复成功！先把业务起来要紧

online所有卷，由于原来的mmdg07磁盘被更换过，这里启动卷过程系统会自动做镜像同步
root@mmsc1 # vxvol -g mmsdg start all  
手工mount卷
mount ...(此处省略一些mount操作)

尝试拉起业务, 结果正常
root@mmsc1 # scswitch -z -g mms_rg -h mmsc1


接下来把c5t12d0s2重新加入到mmsdg
root@mmsc1 # vxdg -g mmsdg adddisk mmsdg07=c5t12d0   这步没有提示错误，OK！
root@mmsc1 # vxedit -g mmsdg set spare=on mmsdg07

完成了整个操作过程。