在 linux 下 安装oracle 10G RAC asm 记录
-----------------------------------------------------------
环境介绍:
两台 IBM x3850,连接 IBM DS4700 存储
操作系统 LINUX AS4 / RHEL5.2
1、安装 rdca(到IBM 网站上下载rdca多路径源代码进行编译安装)
make
make install
按 rdca 里面的 readme 添加引导条目,并更改为默认引导
vi /boot/grub/grub.conf
default=2
timeout=5
splashimage=(hd0,0)/grub/splash.xpm.gz
hiddenmenu
title Red Hat Enterprise Linux AS (2.6.9-67.ELsmp)
root (hd0,0)
kernel /vmlinuz-2.6.9-67.ELsmp ro root=/dev/VolGroup00/LogVol00 rhgb quiet
initrd /initrd-2.6.9-67.ELsmp.img
title Red Hat Enterprise Linux AS-up (2.6.9-67.EL)
root (hd0,0)
kernel /vmlinuz-2.6.9-67.EL ro root=/dev/VolGroup00/LogVol00 rhgb quiet
initrd /initrd-2.6.9-67.EL.img
title Red Hat Linux (2.6.9-67.ELsmp) with MPP support
root (hd0,0)
kernel /vmlinuz-2.6.9-67.ELsmp ro root=/dev/VolGroup00/LogVol00 rhgb quiet
initrd /mpp-2.6.9-67.ELsmp.img
-----------------------------------------------------------------------
2、必须安装软件包
---- 对于 LINUX as 4
# rpm -q binutils compat-db control-center gcc gcc-c++ glibc glibc-common \
gnome-libs libstdc++ libstdc++-devel make pdksh sysstat xscreensaver
binutils-2.15.92.0.2-15
compat-db-4.1.25-9
control-center-2.8.0-12.rhel4.2
gcc-3.4.4-2
gcc-c++-3.4.4-2
glibc-2.3.4-2.13
glibc-common-2.3.4-2.13
gnome-libs-1.4.1.2.90-44.1
libstdc++-3.4.4-2
libstdc++-devel-3.4.4-2
make-3.80-5
pdksh-5.2.14-30.3
sysstat-5.0.5-1
xscreensaver-4.18-5.rhel4.9
对于RHEL5.0 以上 必须安装软件包
-------------------------------
From RedHat AS5 Disk 1
#rpm -Uvh setarch-2*
#rpm -Uvh make-3*
#rpm -Uvh glibc-2*
#rpm -Uvh libaio-0*
From RedHat AS5 Disk 2
#rpm -Uvh compat-libstdc++-33-3*
#rpm -Uvh compat-gcc-34-3*
#rpm -Uvh compat-gcc-34-c++-3*
#rpm -Uvh gcc-4*
#rpm -Uvh libXp-1* （如果不安装libXp软件包则安装不能启动Oracle10g安装程序）
From RedHat AS5 Disk 3
#rpm -Uvh openmotif-2*
#rpm -Uvh compat-db-4*
--------------------------------------------------------------------------
3、设置安装crs 及oracle 的用户\组及相关目录
groupadd -g 500 oinstall
groupadd -g 501 dba
groupadd -g 502 oper
useradd -u 500 -g oinstall -G dba,oper -d /u01/app/oracle oracle
passwd oracle
mkdir -p /u01/app/oracle/product/10.2.0/db_1
mkdir -p /crs
chown -R oracle:oinstall /u01
chmod -R 775 /u01
chown -R oracle:oinstall /crs
chmod -R 775 /crs
------------------------------------
oracle .bash_profile设置
vi .bash_profile
TMP=/tmp; export TMP
TMPDIR=$TMP; export TMPDIR
ORACLE_BASE=/u01/app/oracle;export ORACLE_BASE
ORACLE_HOME=$ORACLE_BASE/product/10.2.0/db_1;export ORACLE_HOME
ORA_CRS_HOME=/crs
export ORA_CRS_HOME
CRS_HOME=/crs
export CRS_HOME
ORACLE_SID=gzczjoa1;export ORACLE_SID
ORACLE_TERM=xterm;export ORACLE_TERM
PATH=/usr/sbin:$PATH;
PATH=$CRS_HOME/bin:$ORACLE_HOME/bin:$PATH
export PATH
LD_LIBRARY_PATH=$ORACLE_HOME/lib:/lib:/usr/lib;
export LD_LIBRARY_PATH
CLASSPATH=$ORACLE_HOME/JRE:$ORACLE_HOME/jlib:$ORACLE_HOME/rdbms/jlib;
export CLASSPATH
export NLS_LANG=AMERICAN_AMERICA.ZHS16GBK
LANG=en_US
export LANG
if [ \$USER = "oracle" ]; then
if [ \$SHELL = "/bin/ksh" ]; then
ulimit -p 16384
ulimit -n 65536
else
ulimit -u 16384 -n 65536
fi
umask 022
fi
---------------------------------------------------------------------
4、shell limits 设置：
cat >> /etc/security/limits.conf << EOF
oracle soft nproc 2047
oracle hard nproc 16384
oracle soft nofile 1024
oracle hard nofile 65536
EOF
---------------------------------------------------------------------
5、对于oracle 用户的参数限制
cat >> /etc/csh.login << EOF
if ( \$USER == "oracle" ) then
limit maxproc 16384
limit descriptors 65536
umask 022
endif
EOF
----------------------------------------------------------------------
6、ssh 密钥配置
mkdir ~/.ssh
chmod 755 ~/.ssh
usr/bin/ssh-keygen -t rsa
/usr/bin/ssh-keygen -t dsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys
ssh x3850b cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
ssh x3850b cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys
chmod 644 ~/.ssh/authorized_keys
scp ~/.ssh/authorized_keys x3850b:~/.ssh/
chmod 644 ~/.ssh/authorized_keys
第二个节点也要进行上面相关的操作
在x3850a节点上:
ssh x3850a date
ssh x3850b date
ssh x3850a-priv date
ssh x3850b-priv date
在x3850b节点上：
ssh x3850a date
ssh x3850b date
ssh x3850a-priv date
ssh x3850b-priv date
-------------------------------------------------------------------------
7、pam安全设置
cat >> /etc/pam.d/login << EOF
session required /lib/security/pam_limits.so
EOF
-------------------------------------------------------------------------
8、配置 Hangcheck Timer 模块,并把其添加到 rc.local 中
modprobe hangcheck-timer hangcheck_tick=30 hangcheck_margin=180
cat >> /etc/rc.d/rc.local << EOF
modprobe hangcheck-timer hangcheck_tick=30 hangcheck_margin=180
EOF
modprobe -v hangcheck-timer
激活
--------------------------------------------------------------------------
9、vi /etc/hosts 文件配置
127.0.0.1 localhost.localdomain localhost
172.16.69.124 x3850a
172.16.69.125 x3850a-vip
192.168.69.100 x3850a-priv
172.16.69.126 x3850b
172.16.69.127 x3850b-vip
192.168.69.101 x3850b-priv
----------------------------------------------------------------------------
10、内核参数调整
cat >> /etc/sysctl.conf << EOF
kernel.shmall = 2097152
kernel.shmmax = 8536870912 #设置为机器配置内存的一半
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
fs.file-max = 658576
net.ipv4.ip_local_port_range = 1024 65000
net.core.rmem_default = 262144
net.core.wmem_default = 262144
net.core.rmem_max = 1048536
net.core.wmem_max = 1048536
EOF
从 sysctl.conf 中加载内核配置
/sbin/sysctl -p
-----------------------------------------------------------------------------
11、时间服务器的配置 NTP 配置
节点1 x3850a
vi /etc/ntp.conf
restrict 172.16.69.126 mask 255.255.255.0 nomodify notrap
启动服务
service ntpd start
节点二 x3850b
crontab -e
5 * * * * ntpdate 172.16.69.124
测试 时间服务器的更新情况
#ntpdate 172.16.69.124
----------------------------------------------------------------------------
12、linux 下建立raw device
linux AS4 下的raw 设备绑定
先对磁盘进行分区（重要）
fdisk -l
//必须是 /dev/raw/rawX 的格式
raw /dev/raw/raw1 /dev/sdd1
raw /dev/raw/raw2 /dev/sde1
raw /dev/raw/raw3 /dev/sdf1
raw /dev/raw/raw4 /dev/sdg1
raw /dev/raw/raw5 /dev/sdh1
编辑文件：/etc/sysconfig/rawdevices
/dev/raw/raw1 /dev/sdd1
/dev/raw/raw2 /dev/sde1
/dev/raw/raw3 /dev/sdf1
/dev/raw/raw4 /dev/sdg1
/dev/raw/raw5 /dev/sdh1
修改raw device 生成权限
/etc/udev/permissions.d/50-udev.permissions
raw/*:root:disk:0660
修改为
raw/*:oracle:oinstall:0660
---------------在RHEL5.2 中
#vi /etc/udev/rules.d/60-raw.rules
ACTION=="add", ENV{MAJOR}=="8", ENV{MINOR}=="49", RUN+="/bin/raw /dev/raw/raw1 %M %m"
ACTION=="add", ENV{MAJOR}=="8", ENV{MINOR}=="65", RUN+="/bin/raw /dev/raw/raw2 %M %m"
ACTION=="add", ENV{MAJOR}=="8", ENV{MINOR}=="81", RUN+="/bin/raw /dev/raw/raw3 %M %m"
ACTION=="add", ENV{MAJOR}=="8", ENV{MINOR}=="97", RUN+="/bin/raw /dev/raw/raw4 %M %m"
ACTION=="add", ENV{MAJOR}=="8", ENV{MINOR}=="113", RUN+="/bin/raw /dev/raw/raw5 %M %m"
ACTION=="add", KERNEL=="sdd1", RUN+="/bin/raw /dev/raw/raw1 %N"
ACTION=="add", KERNEL=="sde1", RUN+="/bin/raw /dev/raw/raw2 %N"
ACTION=="add", KERNEL=="sdf1", RUN+="/bin/raw /dev/raw/raw3 %N"
ACTION=="add", KERNEL=="sdg1", RUN+="/bin/raw /dev/raw/raw4 %N"
ACTION=="add", KERNEL=="sdh1", RUN+="/bin/raw /dev/raw/raw5 %N"
#vi /etc/rc.local 因为raw 设备在启动时是重新加载的,所以要重新设置 raw 属性
chown oracle:oinstall /dev/raw/raw[3-5]
chown oracle:oinstall /dev/raw/raw[1-2]
chmod 660 /dev/raw/raw*
-------------------------------------------------------------------------
13、ASMlib 配置
ASMLIB 安装 --------- 到 oracle 网站上下载 对应于oracle 内核版本的 asmlib
下面为linux AS4 下的 三个asmlib 软件包
rpm -ivh oracleasm-support-2.1.3-1.el4.i386.rpm
rpm -ivh oracleasm-2.6.9-67.ELsmp-2.0.3-1.i686.rpm
rpm -ivh oracleasmlib-2.0.4-1.el4.i386.rpm
对asmlib 进行相关配置：(用户为oracle ,组为dba,开机自启动,磁盘自动加载)
/etc/init.d/oracleasm configure
通过 asmlib 创建 vol
/etc/init.d/oracleasm createdisk VOL1 /dev/sdi1
/etc/init.d/oracleasm createdisk VOL2 /dev/sdj1
/etc/init.d/oracleasm createdisk VOL3 /dev/sdk1
/etc/init.d/oracleasm createdisk VOL4 /dev/sdl1
/etc/init.d/oracleasm createdisk VOL5 /dev/sdm1
# /etc/init.d/oracleasm listdisks
VOL1
VOL2
VOL3
VOL4
VOL5
在另外一个节点
/etc/init.d/oracleasm scandisks
# /etc/init.d/oracleasm listdisks
VOL1
VOL2
VOL3
VOL4
VOL5
删除 asm 的磁盘的操作
/etc/init.d/oracleasm deletedisk VOL1
/etc/init.d/oracleasm deletedisk VOL2
/etc/init.d/oracleasm deletedisk VOL3
/etc/init.d/oracleasm deletedisk VOL4
/etc/init.d/oracleasm deletedisk VOL5
数据库一般创建两个diskgroup分别为: dgdata（系统数据），dgrecover（archive等）
-------------------------------------------------------------------------
14、clusterware及oracle代码安装注意事项
对于 linux AS4 ，直接用oracle用户在图形界面下运行 ./runInstaller
对于 RHEL5 ，应该运行 ./runInstaller -ignoreSysPrereqs -------- 因为 RHEL5 不是被oracle10g 认证的操作系统，所以忽略版本检查。
crs 相关操作命令：在安装完CRS 之后，crs 默认是自启动的
crsctl disable crs 取消crs 开机自启动
crsctl enable crs 取消crs 开机自启动
-------------------------------------------------------------------------
15、在crs安装失败时,要进行相关卸载重新安装
rm -rf /etc/oracle/*
rm -f /etc/init.d/init.cssd
rm -f /etc/init.d/init.crs
rm -f /etc/init.d/init.crsd
rm -f /etc/init.d/init.evmd
rm -f /etc/rc2.d/K96init.crs
rm -f /etc/rc2.d/S96init.crs
rm -f /etc/rc3.d/K96init.crs
rm -f /etc/rc3.d/S96init.crs
rm -f /etc/rc5.d/K96init.crs
rm -f /etc/rc5.d/S96init.crs
rm -Rf /etc/oracle/scls_scr
rm -f /etc/inittab.crs
[ -e "/etc/inittab.orig" ]&&cp /etc/inittab.orig /etc/inittab
删除 CRS 安装目录的内容
对于2个OCR disk 及 3 个vote disk处理.(一定要做分区)
dd if=/dev/zero of=/dev/sdd1 bs=8192 count=102400
dd if=/dev/zero of=/dev/sde1 bs=8192 count=102400
dd if=/dev/zero of=/dev/sdf1 bs=8192 count=102400
dd if=/dev/zero of=/dev/sdg1 bs=8192 count=102400
dd if=/dev/zero of=/dev/sdh1 bs=8192 count=102400
service rawdevices reload 重新加载 rawdevice，适合于 LINUX AS4
------------------------------------------------------------------------------------
16、问题解决
问题一:
[root@x3850b crs]# ./root.sh
Checking to see if Oracle CRS stack is already configured
Setting the permissions on OCR backup directory
Setting up NS directories
Failed to upgrade Oracle Cluster Registry
解决方法:
对于 OCR 磁盘,不应该直接使用 /dev/sdd 应该先进性分区，在应射到RAW devices
对于之前安装失败时，应该 dd 对应的 /dev/sdd1 分区
--------------------------------------------------------------------------------------------------------------
问题二:
在安装了 powerpath 软件的在使用 asm 时。
1.需要在运行root.sh前oracle打补丁4679769。
---------------------------------------------------------------------------------------------------------------
问题三：
2.rootpre.sh有个报错No OraCM running ，可以忽略。
问题四：
3.0 安装crs将要结束时，在第二个节点运行 root.sh 提示：
Oracle CRS stack installed and running under init(1M)
Running vipca(silent) for configuring nodeapps
The given interface(s), "eth0" is not public. Public interfaces should be used to configure virtual IPs.
解决方法：
在 图形界面（xmanger） root 用户，进入 $crs_home/bin,运行 vipca ，创建 vip，结束后再点击确定按钮。
3.1 error while loading shared libraries: libpthread.so.0: cannot open报错。
[root@x3850b oracle]# /crs/root.sh
Checking to see if Oracle CRS stack is already configured
/etc/oracle does not exist. Creating it now.
Setting the permissions on OCR backup directory
Setting up NS directories
Oracle Cluster Registry configuration upgraded successfully
clscfg: EXISTING configuration version 3 detected.
clscfg: version 3 is 10G Release 2.
Successfully accumulated necessary OCR keys.
Using ports: CSS=49895 CRS=49896 EVMC=49898 and EVMR=49897.
node
:
node 1: x3850a x3850a-priv x3850a
node 2: x3850b x3850b-priv x3850b
clscfg: Arguments check out successfully.NO KEYS WERE WRITTEN. Supply -force parameter to override.
-force is destructive and will destroy any previous cluster
configuration.
Oracle Cluster Registry for cluster has already been initialized
Startup will be queued to init within 90 seconds.
Adding daemons to inittab
Expecting the CRS daemons to be up within 600 seconds.
CSS is active on these nodes.
x3850a
x3850b
CSS is active on all nodes.
Waiting for the Oracle CRSD and EVMD to start
Oracle CRS stack installed and running under init(1M)
Running vipca(silent) for configuring nodeapps
/crs/jdk/jre//bin/java: error while loading shared libraries: libpthread.so.0: cannot open shared object file: No such file or directory
解决方法：
首先设置网卡类型
#oifcfg setif –global eth0/172.16.69.0:public
#oifcfg setif –global eth1/192.168.69.0:cluster_interconnect
#oifcfg getif
在作VIPCA前，需要修改$CRS_HOME/bin/vipca文件。具体先查查网上， unset LD_ASSUME_KERNEL
再在图形界面上 运行 vipca 创建服务ip
----------------------------------------------------------------------------------------------------------------------------
问题五：
4.2台机器的时间问题.
启用ntpd 时间服务器解决问题
----------------------------------------------------------------------------------------------------------------------------
问题六：
打补丁时注意事项：
clusterware 的补丁在 oracle patch 里面，应该第一次选择 把patch 打在 crs_home
再进行以下操作：后再打oracle 10g补丁
The installer has detected that your Cluster Ready Services (CRS) installation is distributed across the following nodes:
x3850a
x3850b
Because the software consists of local identical copies distributed across each of the nodes in the cluster,
it is possible to patch your CRS installation in a rolling manner, one node at a time.
To complete the installation of this patchset, you must perform the following tasks on each node:
1. Log in as the root user.
2. As the root user, perform the following tasks:
a. Shutdown the CRS daemons by issuing the following command:
/crs/bin/crsctl stop crs
b. Run the shell script located at:
/crs/install/root102.sh
This script will automatically start the CRS daemons on the
patched node upon completion.
3. After completing this procedure, proceed to the next node and repeat.
--------------------------------------------------------------------------------------------------------------------
问题七：
数据库连接出现时，有些连接能连上，有些不能能连接的现象。
报错提示：目标或主机不存在
解决：
设置数据库参数:
sql> show parameter listener 查看 local_listener 及 remote_listener 的设置
sql> alter system set local_listener='(ADDRESS = (PROTOCOL = TCP)(HOST = 172.16.69.125)(PORT = 1521))' scope=both sid='*';
sql>alter system set remote_listener=LISTENERS_GZCZJOA scope=both sid='*'
///////////////////////////////////////////////tnsname.ora /////////////////////////////////////////
[oracle@x3850b admin]$ cat tnsnames.ora
# tnsnames.ora.x3850b Network Configuration File: /u01/app/oracle/product/10.2.0/db_1/network/admin/tnsnames.ora.x3850b
# Generated by Oracle configuration tools.
GZCZJ_TAF2 =
(DESCRIPTION =
(ADDRESS = (PROTOCOL = TCP)(HOST = 172.16.69.125)(PORT = 1521))
(ADDRESS = (PROTOCOL = TCP)(HOST = 172.16.69.127)(PORT = 1521))
(LOAD_BALANCE = yes)
(CONNECT_DATA =
(SERVER = DEDICATED)
(SERVICE_NAME = gzczj_taf2)
(FAILOVER_MODE =
(TYPE = SELECT)
(METHOD = BASIC)
(RETRIES = 180)
(DELAY = 5)
)
)
)
GZCZJ_TAF1 =
(DESCRIPTION =
(ADDRESS = (PROTOCOL = TCP)(HOST = 172.16.69.125)(PORT = 1521))
(ADDRESS = (PROTOCOL = TCP)(HOST = 172.16.69.127)(PORT = 1521))
(LOAD_BALANCE = yes)
(CONNECT_DATA =
(SERVER = DEDICATED)
(SERVICE_NAME = gzczj_taf1)
(FAILOVER_MODE =
(TYPE = SELECT)
(METHOD = BASIC)
(RETRIES = 180)
(DELAY = 5)
)
)
)
GZCZJOA2 =
(DESCRIPTION =
(ADDRESS = (PROTOCOL = TCP)(HOST = 172.16.69.127)(PORT = 1521))
(CONNECT_DATA =
(SERVER = DEDICATED)
(SERVICE_NAME = gzczjoa)
(INSTANCE_NAME = gzczjoa2)
)
)
GZCZJOA1 =
(DESCRIPTION =
(ADDRESS = (PROTOCOL = TCP)(HOST = 172.16.69.125)(PORT = 1521))
(CONNECT_DATA =
(SERVER = DEDICATED)
(SERVICE_NAME = gzczjoa)
(INSTANCE_NAME = gzczjoa1)
)
)
GZCZJOA =
(DESCRIPTION =
(ADDRESS = (PROTOCOL = TCP)(HOST = 172.16.69.125)(PORT = 1521))
(ADDRESS = (PROTOCOL = TCP)(HOST = 172.16.69.127)(PORT = 1521))
(LOAD_BALANCE = yes)
(CONNECT_DATA =
(SERVER = DEDICATED)
(SERVICE_NAME = gzczjoa)
)
)
LISTENERS_GZCZJOA =
(ADDRESS_LIST =
(ADDRESS = (PROTOCOL = TCP)(HOST = 172.16.69.125)(PORT = 1521))
(ADDRESS = (PROTOCOL = TCP)(HOST = 172.16.69.127)(PORT = 1521))
)
[oracle@x3850a admin]$ cat listener.ora
# listener.ora.x3850a Network Configuration File: /u01/app/oracle/product/10.2.0/db_1/network/admin/listener.ora.x3850a
# Generated by Oracle configuration tools.
[oracle@x3850b ~]$ cat /u01/app/oracle/product/10.2.0/db_1/network/admin/listener.ora
# listener.ora.x3850b Network Configuration File: /u01/app/oracle/product/10.2.0/db_1/network/admin/listener.ora.x3850b
# Generated by Oracle configuration tools.
LISTENER_X3850B =
(DESCRIPTION_LIST =
(DESCRIPTION =
(ADDRESS_LIST =
(ADDRESS = (PROTOCOL = TCP)(HOST = x3850b-vip)(PORT = 1521)(IP = FIRST))
)
(ADDRESS_LIST =
(ADDRESS = (PROTOCOL = TCP)(HOST = 172.16.69.126)(PORT = 1521)(IP = FIRST))
)
(ADDRESS_LIST =
(ADDRESS = (PROTOCOL = IPC)(KEY = EXTPROC))
)
)
)
-------------------------------------------------------------------------
crs 状态
[oracle@x3850b ~]$ crs_stat -t
Name Type Target State Host
------------------------------------------------------------
ora.gzczjoa.db application ONLINE ONLINE x3850a
ora....taf1.cs application ONLINE ONLINE x3850b
ora....oa2.srv application ONLINE ONLINE x3850a
ora....taf2.cs application ONLINE ONLINE x3850a
ora....oa1.srv application ONLINE ONLINE x3850a
ora....a1.inst application ONLINE ONLINE x3850a
ora....a2.inst application ONLINE ONLINE x3850b
ora....SM1.asm application ONLINE ONLINE x3850a
ora....0A.lsnr application ONLINE ONLINE x3850a
ora.x3850a.gsd application ONLINE ONLINE x3850a
ora.x3850a.ons application ONLINE ONLINE x3850a
ora.x3850a.vip application ONLINE ONLINE x3850a
ora....SM2.asm application ONLINE ONLINE x3850b
ora....0B.lsnr application ONLINE ONLINE x3850b
ora.x3850b.gsd application ONLINE ONLINE x3850b
ora.x3850b.ons application ONLINE ONLINE x3850b
ora.x3850b.vip application ONLINE ONLINE x3850b
