Oracle 12cR2 RAC集群安装指南

一、环境介绍

服务器	vSphere 6虚拟机
操作系统	OEL 7.0 with Kernal 3.8.13-35.3.1.el7uek
硬件配置	CPU 32 Cores 内存 96 G 硬盘 150 G

二、IP规划

主机名	Public IP	Private IP	VIP	Scan IP
oraclerac8281	10.116.82.81	192.168.82.81	10.116.82.84	10.116.82.87
oraclerac8282	10.116.82.82	192.168.82.82	10.116.82.85
oraclerac8283	10.116.82.83	192.168.82.83	10.116.82.86

三、存储规划

硬盘	ASM Disk类型	大小
/dev/sdb	OCR/Voting Disk/GIMR	50G
/dev/sdc	OCR/Voting Disk/GIMR	50G
/dev/sdd	OCR/Voting Disk/GIMR	50G
/dev/sde	DATA	2T
/dev/sdf	RECOVERY	1T

四、系统配置（每个节点）

1、禁用Selinux/Firewall

sed -i '/^SELINUX=.*/ s//SELINUX=disabled/' /etc/selinux/config

echo "Disable Firewall"

systemctl disable firewalld

2、配置本地Yum库

cat > /etc/yum.repos.d/local.repo <

[rhel-media]

name=RHEL-\$releasever - Media

baseurl=file:///media

enabled=1

gpgcheck=0

gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release

!

3、配置Host解析

sed -i 3,\$d /etc/hosts

cat >> /etc/hosts <

#Public IP

10.116.82.81     oraclerac8281

10.116.82.82     oraclerac8282

10.116.82.83     oraclerac8283

#Private IP

192.168.82.81    oraclerac8281-priv

192.168.82.82    oraclerac8282-priv

192.168.82.83    oraclerac8283-priv

#VIP

10.116.82.84     oraclerac8281-vip

10.116.82.85     oraclerac8282-vip

10.116.82.86     oraclerac8283-vip

#Scan-IP

10.116.82.87     oraclerac-act-scan

!

4、依赖包安装

yum -y install binutils compat-libcap1 compat-libstdc++-33 \

 compat-libstdc++-33*.i686 gcc gcc-c++ glibc glibc*.i686 \

 glibc-devel glibc-devel*.i686 ksh libaio libaio-*.i686 \

 libaio-devel libaio-devel*.i686 libgcc libgcc*.i686 \

 libstdc++ libstdc++-*.i686 libstdc++-devel libstdc++devel*.i686 \

 libXi libXi*.i686 libXtst libXtst*.i686 make sysstat \

 unixODBC unixODBC-devel unixODBC*.i686 zlib-devel zlib-devel.i686 \

 nfs-utils sysstat chrony smartmontools net-tools zip unzip

5、系统参数配置

MEMTOTAL=$(free -b | sed -n '2p' | awk '{print $2}')

let SHMMAX=$MEMTOTAL*4/5

SHMMNI=4096

let SHMALL=$MEMTOTAL/4096

echo SHMMAX:$SHMMAX SHMALL:$SHMALL

cat > /etc/sysctl.d/98-oracle.conf <

fs.aio-max-nr = 1048576

fs.file-max = 6815744

kernel.shmall = $SHMALL

kernel.shmmax = $SHMMAX

kernel.shmmni = 4096

kernel.sem = 250 32000 100 128

net.ipv4.ip_local_port_range = 9000 65500

net.core.rmem_default = 262144

net.core.rmem_max = 4194304

net.core.wmem_default = 262144

net.core.wmem_max = 1048576

kernel.panic_on_oops = 1

net.ipv4.conf.all.rp_filter = 2

net.ipv4.conf.default.rp_filter = 2

!

/sbin/sysctl -p /etc/sysctl.d/98-oracle.conf

6、添加用户和组

groupadd -g 54321 oinstall

groupadd -g 54322 dba

groupadd -g 54323 oper

groupadd -g 54324 backupdba

groupadd -g 54325 dgdba

groupadd -g 54326 kmdba

groupadd -g 54327 asmdba

groupadd -g 54328 asmoper

groupadd -g 54329 asmadmin

groupadd -g 54330 racdba

useradd -u 54321 -g oinstall -G dba,asmdba,backupdba,dgdba,kmdba,racdba,oper oracle

useradd -u 54331 -g oinstall -G dba,oper,backupdba,dgdba,kmdba,asmdba,asmoper,asmadmin,racdba grid

echo "grid" | passwd --stdin grid

echo "oracle:oracle" | chpasswd

7、创建目录并设置权限

mkdir -p /u01/app/grid

mkdir -p /u01/app/12.2.0/grid

mkdir -p /u01/app/oracle

mkdir -p /u01/app/oracle/product/12.2.0/db_1

mkdir -p /u01/app/oraInventory

chown -R grid:oinstall /u01/app

chown -R oracle:oinstall /u01/app/oracle

chmod -R 775 /u01/app

8、修改用户安全限制

cat > /etc/security/limits.d/oracle-database-server-12cR2-preinstall.conf <

oracle     soft     nofile   2048

oracle     hard     nofile   65536

oracle     soft     nproc    16384

oracle     hard     nproc    16384

oracle     soft     stack    10243

oracle     hard     stack    32768

oracle     hard     memlock  134217728

oracle     soft     memlock  134217728

grid       soft     nofile   2048

grid       hard     nofile   65536

grid       soft     nproc    16384

grid       hard     nproc    16384

grid       soft     stack    10243

grid       hard     stack    32768

grid       hard     memlock  134217728

grid       soft     memlock  134217728
!

9、添加PAM模块

cat >> /etc/pam.d/login <

session   required    pam_limits.so

!

10、禁用透明大页

cat >> /etc/rc.local <

if test -f /sys/kernel/mm/redhat_transparent_hugepage/enabled; then

  echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled

fi

!

11、配置Grid环境变量

sed -i 13,\$d ~grid/.bash_profile

cat >> ~grid/.bash_profile <

export ORACLE_BASE=/u01/app/grid

export ORACLE_HOME=/u01/app/12.2.0/grid

export ORACLE_SID=+ASM$(STR=$(hostname); echo ${STR: -1})

export PATH=\$ORACLE_HOME/bin:\$ORACLE_HOME/OPatch:\$PATH

export LD_LIBRARY_PATH=\$ORACLE_HOME/lib:\$LD_LIBRARY_PATH

!

cat ~grid/.bash_profile

12、配置Oracle环境变量

sed -i 13,\$d ~oracle/.bash_profile

cat >> ~oracle/.bash_profile <

export ORACLE_BASE=/u01/app/oracle

export ORACLE_HOME=\$ORACLE_BASE/product/12.2.0/db_1

export ORACLE_SID=orcl1$(STR=$(hostname); echo ${STR: -1})

export NLS_LANG=AMERICAN_AMERICA.AL32UTF8

export PATH=\$ORACLE_HOME/bin:\$ORACLE_HOME/OPatch:\$PATH

export LD_LIBRARY_PATH=\$ORACLE_HOME/lib:\$LD_LIBRARY_PATH

!

cat ~oracle/.bash_profile

13、配置Oracle/Grid互信（只需在一个节点执行）

Oracle提供的database安装包里有脚本可以配置互信，比较方便。

su - oracle -c "unzip -qo /tmp/linuxx64_12201_database.zip -d /u01/app"

/u01/app/database/sshsetup/sshUserSetup.sh -user oracle -hosts "oraclerac8281 oraclerac8282 oraclerac8283" -advanced -noPromptPassphrase

/u01/app/database/sshsetup/sshUserSetup.sh -user grid -hosts "oraclerac8281 oraclerac8282 oraclerac8283" -advanced -noPromptPassphrase

配置完成后使用下面命令测试用户连通性

su - oracle -c "ssh oraclerac8281 date; ssh oraclerac8282 date; ssh oraclerac8283 date; exit"

su - grid -c "ssh oraclerac8281 date; ssh oraclerac8282 date; ssh oraclerac8283 date; exit"

分发cvuqdisk安装包到所有节点/tmp下

su - oracle -c "

scp /u01/app/database/rpm/cvuqdisk-1.0.10-1.rpm oraclerac8281:/tmp

scp /u01/app/database/rpm/cvuqdisk-1.0.10-1.rpm oraclerac8282:/tmp

scp /u01/app/database/rpm/cvuqdisk-1.0.10-1.rpm oraclerac8283:/tmp

"

14、配置NOZEROCONF

echo "NOZEROCONF=yes" >> /etc/sysconfig/network

grep ^NOZEROCONF /etc/sysconfig/network

15、修改nsswitch.conf

sed -i "/^hosts:/chosts:      files dns nis" /etc/nsswitch.conf

grep ^hosts /etc/nsswitch.conf

16、配置tmpfs挂载大小

默认tmpfs挂载的大小为内存的一半，SGA设置如果超过内存一半，需要重新配置。

mem=$(free -m | grep ^Mem | awk '{print$2}'); let sga=mem/100*64

sed -i /^tmpfs/d /etc/fstab

echo "tmpfs /dev/shm tmpfs size=${sga}M,mode=1777,nosuid,nodev,strictatime 0 0" >> /etc/fstab

mount -o loop /dev/shm

df -h /dev/shm

17、安装CVU Disk

export CVUQDISK_GRP=asmadmin

rpm -ivh /tmp/cvuqdisk-1.0.10-1.rpm

18、桌面环境安装（只需要装一个节点）

如果RAC服务器和本地节点全端口互通，只需要安装X Window即可，否则需要安装桌面环境

X Window安装"（可选）

yum -y groupinstall "X Window system"

Gnome桌面环境安装（可选）

yum -y groupinstall "Server with GUI"

五、UDEV配置

1、获取Disk UUID

虚拟机需要关机，设置disk.EnableUUID=TRUE。

虚拟机属性->选项->高级->常规->配置参数

2、配置UDEV Rule文件

使用上一步获取到的UUID一一对应替换下面RESULT值，并自定义挂载名称（标红部分）。

cat > /etc/udev/rules.d/99-oracle-asmdevices.rules <

KERNEL=="sd*", ENV{DEVTYPE}=="disk", SUBSYSTEM=="block", PROGRAM=="/usr/lib/udev/scsi_id -g -u -d \$devnode", RESULT=="36000c29d6d2dd44e3dc77ec91b3c4b27", RUN+="/bin/sh -c 'mknod /dev/asmdisk01 b \$major \$minor; chown grid:asmadmin /dev/asmdisk01; chmod 0660 /dev/asmdisk01'"

KERNEL=="sd*", ENV{DEVTYPE}=="disk", SUBSYSTEM=="block", PROGRAM=="/usr/lib/udev/scsi_id -g -u -d \$devnode", RESULT=="36000c29f112c5cc79f547b0571cf17e6", RUN+="/bin/sh -c 'mknod /dev/asmdisk02 b \$major \$minor; chown grid:asmadmin /dev/asmdisk02; chmod 0660 /dev/asmdisk02'"

KERNEL=="sd*", ENV{DEVTYPE}=="disk", SUBSYSTEM=="block", PROGRAM=="/usr/lib/udev/scsi_id -g -u -d \$devnode", RESULT=="36000c29e113772582ecb184c62e6f470", RUN+="/bin/sh -c 'mknod /dev/asmdisk03 b \$major \$minor; chown grid:asmadmin /dev/asmdisk03; chmod 0660 /dev/asmdisk03'"

KERNEL=="sd*", ENV{DEVTYPE}=="disk", SUBSYSTEM=="block", PROGRAM=="/usr/lib/udev/scsi_id -g -u -d \$devnode", RESULT=="36000c290e51112c33800622a77a3cc0e", RUN+="/bin/sh -c 'mknod /dev/datdisk01 b \$major \$minor; chown grid:asmadmin /dev/datdisk01; chmod 0660 /dev/datdisk01'"

KERNEL=="sd*", ENV{DEVTYPE}=="disk", SUBSYSTEM=="block", PROGRAM=="/usr/lib/udev/scsi_id -g -u -d \$devnode", RESULT=="36000c29783cf6791a19b69403dd9a064", RUN+="/bin/sh -c 'mknod /dev/arcdisk01 b \$major \$minor; chown grid:asmadmin /dev/arcdisk01; chmod 0660 /dev/arcdisk01'"

!

cat >> /etc/udev/rules.d/99-oracle-asmdevices.rules <

ACTION=="add|change", ENV{ID_SERIAL}=="36000c29d6d2dd44e3dc77ec91b3c4b27", NAME="asmdisk01",  OWNER="grid", GROUP="asmadmin", MODE="0660"

ACTION=="add|change", ENV{ID_SERIAL}=="36000c29f112c5cc79f547b0571cf17e6", NAME="asmdisk02",  OWNER="grid", GROUP="asmadmin", MODE="0660"

ACTION=="add|change", ENV{ID_SERIAL}=="36000c29e113772582ecb184c62e6f470", NAME="asmdisk03",  OWNER="grid", GROUP="asmadmin", MODE="0660"

ACTION=="add|change", ENV{ID_SERIAL}=="36000c290e51112c33800622a77a3cc0e", NAME="datdisk01",  OWNER="grid", GROUP="asmadmin", MODE="0660"

ACTION=="add|change", ENV{ID_SERIAL}=="36000c29783cf6791a19b69403dd9a064", NAME="arcdisk01",  OWNER="grid", GROUP="asmadmin", MODE="0660"

!

3、重新加载规则

udevadm control --reload-rules

/sbin/udevadm trigger --type=devices --action=change

sleep 3

ls -Ll /dev/sd*

ls -Ll /dev/*disk0*

确认Owner/Group正确。

# ls -Ll /dev/sd*

brw-rw---- 1 root disk     8,  0 Apr 24  2020 /dev/sda

brw-rw---- 1 root disk     8,  1 Apr 24  2020 /dev/sda1

brw-rw---- 1 root disk     8,  2 Apr 24  2020 /dev/sda2

brw-rw---- 1 grid asmadmin 8, 16 Apr 24  2020 /dev/sdb

brw-rw---- 1 grid asmadmin 8, 32 Apr 24  2020 /dev/sdc

brw-rw---- 1 grid asmadmin 8, 48 Apr 24  2020 /dev/sdd

brw-rw---- 1 grid asmadmin 8, 64 Apr 24  2020 /dev/sde

brw-rw---- 1 grid asmadmin 8, 80 Apr 24  2020 /dev/sdf

# ls /dev/*disk0* -l

brw-rw---- 1 grid asmadmin 8, 80 Apr 23 22:35 /dev/arcdisk01

brw-rw---- 1 grid asmadmin 8, 16 Apr 23 22:35 /dev/asmdisk01

brw-rw---- 1 grid asmadmin 8, 32 Apr 23 22:35 /dev/asmdisk02

brw-rw---- 1 grid asmadmin 8, 48 Apr 23 22:35 /dev/asmdisk03

brw-rw---- 1 grid asmadmin 8, 64 Apr 23 22:35 /dev/datdisk01

插曲：一定要确保原始盘的属主配置正确，否则在GI执行root.sh第18步的时候会出现CLSRSC-182: Failed to add the resource for ASM diskgroup的错误。

六、安装Grid Infrastructure（节点1执行）

1、解压

因为从12c R2开始，Grid组件以解压安装的方式，所以需要将压缩包直接解压到$ORACLE_HOME目录，否则解压的目录就是安装目录了。

unzip -q /tmp/linuxx64_12201_grid_home.zip -d $ORACLE_HOME

2、检测安装环境

$ORACLE_HOME/runcluvfy.sh  stage -pre crsinst -n oraclerac8281,oraclerac8282,oraclerac8283 -fixup -verbose

3、开始安装

$ORACLE_HOME/gridSetup.sh

4、执行脚本（root）

/u01/app/oraInventory/orainstRoot.sh

/u01/app/12.2.0/grid/root.sh

等节点1执行成功之后，在其它节点并行执行。

5、检查集群状态

crsctl status res -t

6、创建其它磁盘组

asmca

七、安装Database（节点1执行）

1、安装

/u01/database/runInstaller

2、执行脚本（root）

/u01/app/oracle/product/12.2.0/db_1/root.sh

3、检查数据库集群状态

srvctl status database -d orcl

附录

1、ORACLE 12.2认证的OS软件版本

序号	组件
1	Oracle Linux 7 with the Unbreakable Enterprise kernel: 3.8.13-35.3.1.el7uek.x86_64 or later
2	Oracle Linux 7 with the Red Hat Compatible kernel: 3.10.0-123.el7.x86_64 or later
3	Oracle Linux 6.4 with the Unbreakable Enterprise kernel 2: 2.6.39-200.24.1.el6uek.x86_64 or later
4	Oracle Linux 6 with the Red Hat Compatible kernel: 2.6.32-358.el6.x86_64 or later
5	Red Hat Enterprise Linux 7: 3.10.0-123.el7.x86_64 or later
6	Red Hat Enterprise Linux 6: 2.6.32-358.el6.x86_64 or later
7	SUSE Linux Enterprise Server 12: 3.12.28-4-default or later
8	SUSE Linux Enterprise Server 11 SP2: 3.0.13-0.27 or later

2、集群管理命令

启动Cluster集群(root)

/u01/app/12.2.0/grid/bin/crsctl start cluster [-all | -n ]

关闭Cluster集群(root)

/u01/app/12.2.0/grid/bin/crsctl start cluster [-all | -n ]

-all              启动集群内所有节点

-n  启动指定节点

查看集群状态(grid)

crsctl status res -t

查看数据库集群状态(oracle)

srvctl status database -d orcl

查看Listener状态

srvctl status listener

关闭实例

srvctl stop instance -d orcl -n

启动实例

srvctl start instance -d orcl -n

关闭数据库

srvctl stop database -d orcl

启动数据库

srvctl start database -d orcl