一、描述：部署基于ISCSI方案的GFS。
涉及技术：
并发文件系统技术 --GFS2
基于IP协议的SAN技术 --ISCSI
RedHat 集群技术 --RHCS
二、环境：
一台iscsi—target ：
采用centos 5.2 x64 的系统，采用PXE方式安装8G系统硬盘，外加24G硬盘；
作为iscsi的target服务端，把本地硬盘上的分区通过iscsi共享给iscsi的initiator客户端；
两台iscsi—initiator：
entos 5.2 x64 的系统，采用PXE方式安装，各有一块8G系统硬盘；
作为iscsi的initiator客户端，连接iscsi-target服务端的硬盘，并在集群中规划为GFS文件系统模式下的集群存储池；
三、IP配置：
#iscsi—target：
IP：10.0.40.31
DNS：target.haoyuan-inc.com
#iscsi—initiator：
IP：10.0.70.51
DNS：rhcs1.haoyuan-inc.com
#iscsi—initiator：
IP：10.0.70.61
DNS：rhcs2.haoyuan-inc.com
/etc/hosts 文件配置
# Do not remove the following line, or various programs
# that require network functionality will fail.
127.0.0.1 localhost.localdomain localhost
::1 localhost6.localdomain6 localhost6
10.0.40.31 target.haoyuan-inc.com target
10.0.70.51 rhcs1.haoyuan-inc.com rhcs1
10.0.70.61 rhcs2.haoyuan-inc.com rhcs2
四、ISCSI—Target(10.0.40.31)配置步骤：
4.1 安装iscsi软件
#下载iSCSI Enterprise Target
wget
#解压缩
tar -zxvf iscsitarget-0.4.17.tar.gz
cd iscsitarget-0.4.17
#编译、安装
make
make install
#或者使用yum安装
Yum install -y scsi-target-utils
#安装集群软件包套件
yum groupinstall "Clustering" "Cluster Storage"
#配置服务
chkconfig --level 2345 tgtd on
service tdtd start
4.2挂载设备
#配置第一个iSCSI设备
#系统另外挂了一个24G的硬盘，划分出三个分区即：
[root@10_0_40_31 ~]# fdisk -l
Disk /dev/sda: 8589 MB, 8589934592 bytes
255 heads, 63 sectors/track, 1044 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sda1 * 1 521 4184901 83 Linux
/dev/sda2 522 1043 4192965 82 Linux swap / Solaris
Disk /dev/sdb: 25.7 GB, 25769803776 bytes
255 heads, 63 sectors/track, 3133 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sdb1 1 1045 8393931 83 Linux
/dev/sdb2 1046 2090 8393962+ 83 Linux
/dev/sdb3 2091 3133 8377897+ 83 Linux
#通过tgt服务器把这三个分区共享出来：
[root@10_0_40_31 ~]# chkconfig tgtd on
[root@10_0_40_31 ~]# service tgtd restart
Stopping SCSI target daemon: [ OK ]
Starting SCSI target daemon: [ OK ]
#使用tgtadm定义iscsi target的qualified的名字：
[root@10_0_40_31 ~]# tgtadm --lld iscsi --op new --mode target --tid 1 -T iqn.2009-08.com.haoyuan-inc.disk1
[root@10_0_40_31 ~]# tgtadm --lld iscsi --op new --mode target --tid 2 -T iqn.2009-08.com.haoyuan-inc.disk2
[root@10_0_40_31 ~]# tgtadm --lld iscsi --op new --mode target --tid 3 -T iqn.2009-08.com.haoyuan-inc.disk3
#使用tgtadm为上一步创建的目标增加分区：
[root@10_0_40_31 ~]# tgtadm --lld iscsi --op new --mode logicalunit --tid 1 --lun 1 -b /dev/sdb1
[root@10_0_40_31 ~]# tgtadm --lld iscsi --op new --mode logicalunit --tid 2 --lun 1 -b /dev/sdb2
[root@10_0_40_31 ~]# tgtadm --lld iscsi --op new --mode logicalunit --tid 3 --lun 1 -b /dev/sdb3
#使用tgtadm允许客户端访问这三个目标逻辑卷：
[root@10_0_40_31 ~]# tgtadm --lld iscsi --op bind --mode target --tid 1 -I ALL
[root@10_0_40_31 ~]# tgtadm --lld iscsi --op bind --mode target --tid 2 -I ALL
[root@10_0_40_31 ~]# tgtadm --lld iscsi --op bind --mode target --tid 3 -I ALL
#使用tatadm验证所有的目标逻辑卷定义正确：
[root@10_0_40_31 ~]# tgtadm --lld iscsi --op show --mode target |grep Target
Target 1: iqn.2009-08.com.haoyuan-inc.disk1
Target 2: iqn.2009-08.com.haoyuan-inc.disk2
Target 3: iqn.2009-08.com.haoyuan-inc.disk3
#为使配置生效，将上述命令添加到/etc/rc.local,添加命令：
cat >> /etc/rc.local << EOF
# tgtadm --lld iscsi --op new --mode target --tid 1 -T iqn.2009-08.com.haoyuan-inc.disk1
# tgtadm --lld iscsi --op new --mode target --tid 2 -T iqn.2009-08.com.haoyuan-inc.disk2
# tgtadm --lld iscsi --op new --mode target --tid 3 -T iqn.2009-08.com.haoyuan-inc.disk3
# tgtadm --lld iscsi --op new --mode logicalunit --tid 1 --lun 1 -b /dev/sdb1
# tgtadm --lld iscsi --op new --mode logicalunit --tid 2 --lun 1 -b /dev/sdb2
# tgtadm --lld iscsi --op new --mode logicalunit --tid 3 --lun 1 -b /dev/sdb3
# tgtadm --lld iscsi --op bind --mode target --tid 1 -I ALL
# tgtadm --lld iscsi --op bind --mode target --tid 2 -I ALL
# tgtadm --lld iscsi --op bind --mode target --tid 3 -I ALL
EOF
五、ISCSI—Initiator(10.0.70.51&10.0.70.61)配置步骤：
5.1 rhcs1配置步骤：
#安装iscsi-initiator软件包
Yum install -y iscsi-initiator-utils
#安装集群软件包套件
yum groupinstall "Clustering" "Cluster Storage"
#启动iscsifuw
[root@10_0_70_51 ~]# chkconfig iscsi on
[root@10_0_70_51 ~]# service iscsi start
iscsid is stopped
Turning off network shutdown. Starting iSCSI daemon: [ OK ]
[ OK ]
Setting up iSCSI targets: iscsiadm: No records found!
[ OK ]
#运行下面命令，查看target上的逻辑卷
[root@10_0_70_51 ~]# iscsiadm -m discovery -t sendtargets -p 10.0.40.31
10.0.40.31:3260,1 iqn.2009-08.com.haoyuan-inc.disk1
10.0.40.31:3260,1 iqn.2009-08.com.haoyuan-inc.disk2
10.0.40.31:3260,1 iqn.2009-08.com.haoyuan-inc.disk3
#上述表示已经挂载成功
#使用iscsiadmin登陆target：
[root@10_0_70_51 ~]# iscsiadm -m node -T iqn.2009-08.com.haoyuan-inc.disk1 -p 10.0.40.31 -l
Logging in to [iface: default, target: iqn.2009-08.com.haoyuan-inc.disk1, portal: 10.0.40.31,3260]
Login to [iface: default, target: iqn.2009-08.com.haoyuan-inc.disk1, portal: 10.0.40.31,3260]: successful
[root@10_0_70_51 ~]# iscsiadm -m node -T iqn.2009-08.com.haoyuan-inc.disk2 -p 10.0.40.31 -l
Logging in to [iface: default, target: iqn.2009-08.com.haoyuan-inc.disk2, portal: 10.0.40.31,3260]
Login to [iface: default, target: iqn.2009-08.com.haoyuan-inc.disk2, portal: 10.0.40.31,3260]: successful
[root@10_0_70_51 ~]# iscsiadm -m node -T iqn.2009-08.com.haoyuan-inc.disk3 -p 10.0.40.31 -l
Logging in to [iface: default, target: iqn.2009-08.com.haoyuan-inc.disk3, portal: 10.0.40.31,3260]
Login to [iface: default, target: iqn.2009-08.com.haoyuan-inc.disk3, portal: 10.0.40.31,3260]: successfu
#使用fdisk检验被识别的设备名：
[root@10_0_70_51 ~]# fdisk -l
Disk /dev/sda: 8589 MB, 8589934592 bytes
255 heads, 63 sectors/track, 1044 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sda1 * 1 521 4184901 83 Linux
/dev/sda2 522 1043 4192965 82 Linux swap / Solaris
Disk /dev/sdb: 8595 MB, 8595385344 bytes
64 heads, 32 sectors/track, 8197 cylinders
Units = cylinders of 2048 * 512 = 1048576 bytes
Disk /dev/sdb doesn't contain a valid partition table
Disk /dev/sdc: 8595 MB, 8595417600 bytes
64 heads, 32 sectors/track, 8197 cylinders
Units = cylinders of 2048 * 512 = 1048576 bytes
Disk /dev/sdc doesn't contain a valid partition table
Disk /dev/sdd: 8578 MB, 8578967040 bytes
64 heads, 32 sectors/track, 8181 cylinders
Units = cylinders of 2048 * 512 = 1048576 bytes
Disk /dev/sdd doesn't contain a valid partition table
#从上面看，iscsitarget上的逻辑卷，已经被识别为 /dev/sdb /dev/sdc /dev/sdc三个本地硬盘
#在这三个硬盘上分别建立三个分区，下面为结果：
[root@10_0_70_51 ~]# fdisk -l
Disk /dev/sda: 8589 MB, 8589934592 bytes
255 heads, 63 sectors/track, 1044 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sda1 * 1 521 4184901 83 Linux
/dev/sda2 522 1043 4192965 82 Linux swap / Solaris
Disk /dev/sdb: 8595 MB, 8595385344 bytes
64 heads, 32 sectors/track, 8197 cylinders
Units = cylinders of 2048 * 512 = 1048576 bytes
Device Boot Start End Blocks Id System
/dev/sdb1 1 8197 8393712 83 Linux
Disk /dev/sdc: 8595 MB, 8595417600 bytes
64 heads, 32 sectors/track, 8197 cylinders
Units = cylinders of 2048 * 512 = 1048576 bytes
Device Boot Start End Blocks Id System
/dev/sdc1 1 8197 8393712 83 Linux
Disk /dev/sdd: 8578 MB, 8578967040 bytes
64 heads, 32 sectors/track, 8181 cylinders
Units = cylinders of 2048 * 512 = 1048576 bytes
Device Boot Start End Blocks Id System
/dev/sdd1 1 8181 8377328 83 Linux
5.2 rhcs2配置步骤：
#安装iscsi-initiator软件包
Yum install -y iscsi-initiator-utils
#安装集群软件包套件
yum groupinstall "Clustering" "Cluster Storage"
#启动iscsifuw
[root@10_0_70_61 ~]# chkconfig iscsi on
[root@10_0_70_61 ~]# service iscsi start
iscsid is stopped
Turning off network shutdown. Starting iSCSI daemon: [ OK ]
[ OK ]
Setting up iSCSI targets: iscsiadm: No records found!
[ OK ]
#运行下面命令，查看target上的逻辑卷
[root@10_0_70_61 ~]# iscsiadm -m discovery -t sendtargets -p 10.0.40.31
10.0.40.31:3260,1 iqn.2009-08.com.haoyuan-inc.disk1
10.0.40.31:3260,1 iqn.2009-08.com.haoyuan-inc.disk2
10.0.40.31:3260,1 iqn.2009-08.com.haoyuan-inc.disk3
#上述表示已经挂载成功
#使用iscsiadmin登陆target：
[root@10_0_70_61 ~]# iscsiadm -m node -T iqn.2009-08.com.haoyuan-inc.disk1 -p 10.0.40.31 -l
Logging in to [iface: default, target: iqn.2009-08.com.haoyuan-inc.disk1, portal: 10.0.40.31,3260]
Login to [iface: default, target: iqn.2009-08.com.haoyuan-inc.disk1, portal: 10.0.40.31,3260]: successful
[root@10_0_70_61 ~]# iscsiadm -m node -T iqn.2009-08.com.haoyuan-inc.disk2 -p 10.0.40.31 -l
Logging in to [iface: default, target: iqn.2009-08.com.haoyuan-inc.disk2, portal: 10.0.40.31,3260]
Login to [iface: default, target: iqn.2009-08.com.haoyuan-inc.disk2, portal: 10.0.40.31,3260]: successful
[root@10_0_70_61 ~]# iscsiadm -m node -T iqn.2009-08.com.haoyuan-inc.disk3 -p 10.0.40.31 -l
Logging in to [iface: default, target: iqn.2009-08.com.haoyuan-inc.disk3, portal: 10.0.40.31,3260]
Login to [iface: default, target: iqn.2009-08.com.haoyuan-inc.disk3, portal: 10.0.40.31,3260]: successful
#扫描本地硬盘:
[root@10_0_70_61 ~]# fdisk -l
Disk /dev/sda: 8589 MB, 8589934592 bytes
255 heads, 63 sectors/track, 1044 cylinders
Units = cylinders of 16065 * 512 = 8225280 bytes
Device Boot Start End Blocks Id System
/dev/sda1 * 1 521 4184901 83 Linux
/dev/sda2 522 1043 4192965 82 Linux swap / Solaris
Disk /dev/sdb: 8595 MB, 8595385344 bytes
64 heads, 32 sectors/track, 8197 cylinders
Units = cylinders of 2048 * 512 = 1048576 bytes
Device Boot Start End Blocks Id System
/dev/sdb1 1 8197 8393712 83 Linux
Disk /dev/sdc: 8595 MB, 8595417600 bytes
64 heads, 32 sectors/track, 8197 cylinders
Units = cylinders of 2048 * 512 = 1048576 bytes
Device Boot Start End Blocks Id System
/dev/sdc1 1 8197 8393712 83 Linux
Disk /dev/sdd: 8578 MB, 8578967040 bytes
64 heads, 32 sectors/track, 8181 cylinders
Units = cylinders of 2048 * 512 = 1048576 bytes
Device Boot Start End Blocks Id System
/dev/sdd1 1 8181 8377328 83 Linux
六、配置rhcs1和rhcs2分别enable基于集群的lvm-clvm：
#开启集群
Lvmconf --enable-cluster
#添加开机启动
Chkconfig clvmd on
启动clvmd服务
Server clvmd start
[root@10_0_70_51 ~]# lvmconf --enable-cluster
[root@10_0_70_51 ~]# chkconfig clvmd on
[root@10_0_70_51 ~]# service clvmd start
[root@10_0_70_61 ~]# lvmconf --enable-cluster
[root@10_0_70_61 ~]# chkconfig clvmd on
[root@10_0_70_61 ~]# service clvmd start
七、配置GFS和CLUSTER
7.1 基本设置
#初始化luci：
Luci_admin init
#添加开机启动
Chkconfig clvmd on
#重启动 luci服务
Service luci restart
[root@10_0_40_31 ~]# luci_admin init
Initializing the luci server
Creating the 'admin' user
Enter password:
Confirm password:
Please wait...
The admin password has been successfully set.
Generating SSL certificates...
The luci server has been successfully initialized
You must restart the luci server for changes to take effect.
Run "service luci restart" to do so
[root@10_0_40_31 ~]# chkconfig clvmd on
[root@10_0_40_31 ~]# service luci restart
Shutting down luci: [ OK ]
Starting luci: Generating https SSL certificates... done
[ OK ]
Point your web browser to
#登陆luci ，如图：
#如出现Site Error 这样可以解决：
IE中的设置,点击IE菜单栏中的"工具"--"Internet 首选项"---"常规"点击语言按钮,添加英语,将将设为最优先.
7.2 创建hy0-cluster基本框架
#添加机器到集群，如图：
#提交后结果，如图：
#在经过install-reboot-configure-join四个过程后，hy0-cluster基本形成，如图：
#查看hy0-cluster的状态：
#绿色表示正常
#查看运行的服务,cman&rgmanager
7.3 设置共享存储
#点击storage-system-list-rhcs1-haoyuan-inc.com,如图：
#点击Volume Groups 点击New Volume Group 输入名称：HyVol，选择 /dev/sdb /dev/sdc /dev/sdc 点击create ，如图：
#点击New Logical Volume按钮，创建逻辑卷,如图：
Name：HyVol
Content：gfs2
GFS name：Hy-lv
Mountpoint：/lv
Number:3
创建完毕后，结果如下：
#查看
[root@10_0_70_61 mapper]# df -lh
Filesystem Size Used Avail Use% Mounted on
/dev/sda1 3.9G 3.2G 531M 86% /
tmpfs 1006M 0 1006M 0% /dev/shm
/dev/shm 1006M 0 1006M 0% /tmp
/dev/mapper/HyVol-hy_lv
24G 67M 24G 1% /lv
#/etc/fstab
..........
/dev/mapper/HyVol-hy_lv /lv gfs2 defaults 0 0
八、补充说明
以上为luci方式配置的集群，这种方法最为方便，其他方式有向导和命令方式，向导方式和luci方式大致相同暂略，下面为命令行方式配置步骤：
#基本环境：
#iscsi—target：
IP：10.0.40.31
DNS：target.haoyuan-inc.com
#iscsi—initiator：
IP：10.0.70.51
DNS：rhcs1.haoyuan-inc.com
#iscsi—initiator：
IP：10.0.70.61
DNS：rhcs2.haoyuan-inc.com
#安装软件包
[rhcs1 and rhcs2] yum install -y cman gfs-utils kmod-gfs kmod-dlm modcluster ricci cluster-snmp iscsi-initiator-utils lvm2-cluster openais oddjob rgmanager
#配置iscsi-initiator
[rhcs1 and rhcs2] echo -e "node.startup = automatic\nnode.session.auth.username = ISCSI_USER\nnode.session.auth.password = ISCSI_PASS\ndiscovery.sendtargets.auth.username = ISCSI_USER\ndiscovery.sendtargets.auth.password = ISCSI_PASS\nnode.session.timeo.replacement_timeout = 120\nnode.conn[0].timeo.login_timeout = 15\nnode.conn[0].timeo.logout_timeout = 15\nnode.conn[0].timeo.noop_out_interval = 10\nnode.conn[0].timeo.noop_out_timeout = 15\nnode.session.iscsi.InitialR2T = No\nnode.session.iscsi.ImmediateData = Yes\nnode.session.iscsi.FirstBurstLength = 262144\nnode.session.iscsi.MaxBurstLength = 16776192\nnode.conn[0].iscsi.MaxRecvDataSegmentLength = 65536">/etc/iscsi/iscsid.Conf
#ISCSI_USER 更改为你的用户名
#ISCSI_PASS 更改为你的密码
#启动iscsi服务
[rhcs1 and rhcs2] service iscsi start
#查看iscsi-target端列表
[rhcs1 and rhcs2] iscsiadm -m discovery -t sendtargets -p 10.0.40.31
#重启iscsi服务
[rhcs1 and rhcs2] service iscsi restart
#查看结果
[rhcs1 and rhcs2] disk -l
/dev/sdb
/dev/sdc
/dev/sdd
#已经成功挂载iscsi 设备
#配置/etc/hosts文件
[rhcs1 and rhcs2 and iscsi-server] vi /etc/hosts
# Do not remove the following line, or various programs
# that require network functionality will fail.
127.0.0.1 localhost.localdomain localhost
::1 localhost6.localdomain6 localhost6
10.0.40.31 target.haoyuan-inc.com target
10.0.70.51 rhcs1.haoyuan-inc.com rhcs1
10.0.70.61 rhcs2.haoyuan-inc.com rhcs2
#配置节点
[rhcs1 and rhcs2]
ccs_tool create MyGFSCluster
ccs_tool addfence -C node1_ipmi fence_ipmilan ipaddr=192.168.255.1 login=root passwd=密码
ccs_tool addfence -C node2_ipmi fence_ipmilan ipaddr=192.168.255.2 login=root passwd=密码
ccs_tool addnode -C rhcs1 -n 1 -v 1 -f node1_ipmi
ccs_tool addnode -C rhcs2 -n 2 -v 1 -f node2_ipmi
service cman start
cman_tool nodes
Node Sts Inc Joined Name
1 M 4 rhcs1
2 M 16 rhcs2
#启动GFS服务
service gfs start ;
service gfs2 start ;
#启动clvmd服务
service clvmd start
#开启cluster
lvmconf --enable-cluster
#装机GFS文件系统
[rhcs1]
pvcreate /dev/sda
vgcreate vg_test /dev/sda
lvcreate -n GFStest -L 9G vg_test
gfs_mkfs -j 4 -p lock_dlm -t MyGFSCluster:FirstGFSVolume /dev/vg_test/GFStest
#启动相关服务
[rhcs1 and rhcs2]
chkconfig gfs on
chkconfig gfs2 on
chkconfig clvmd on
chkconfig cman on
chkconfig iscsi on
chkconfig acpid off
#挂载逻辑卷
echo "/dev/vg_test/GFStest /mnt gfs defaults 0 0" >>/etc/fstab
mount /mnt
#验证配置结果
[rhcs1]
Filesystem Type Size Used Avail Use% Mounted on
/dev/hda1 ext3 8.8G 695M 7.6G 9% /
tmpfs tmpfs 506M 0 506M 0% /dev/shm
/dev/mapper/gfstest-lvgfstest
gfs 19G 11M 19G 1% /mnt
[rhcs2]
Filesystem Type Size Used Avail Use% Mounted on
/dev/hda1 ext3 8.8G 695M 7.6G 9% /
tmpfs tmpfs 506M 0 506M 0% /dev/shm
/dev/mapper/gfstest-lvgfstest
gfs 19G 11M 19G 1% /mnt
