1、明确任务
准备一台RHEL5的PC机充当SAN设备，san（同时充当DNS和授时系统）
准备两台RHEL5的PC机充当RAC节点，r1和r2
将RAC安装在ASM磁盘上，包括OCR、VOTING、库
2、修改r1和r2的内核参数与shell限制
在r1和r2上分别进行
vim /etc/sysctl.conf
------------------------------------------
kernel.shmmax
= 4294967296
kernel.shmmni = 4096
kernel.shmall = 2097152
kernel.sem =
250 32000 100 128
fs.file-max = 6815744
fs.aio-max-nr =
1048576
net.ipv4.ip_local_port_range = 9000
65500
net.core.rmem_default = 262144
net.core.rmem_max =
4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048586
sysctl -p
vim
/etc/security/limits.conf
------------------------------------------
#Oracle configure shell parameters
oracle soft nofile
65536
oracle hard nofile 65536
oracle soft nproc 16384
oracle hard nproc
16384
grid soft nofile 65536
grid hard
nofile 65536
grid soft nproc 16384
grid hard nproc 16384
3、配置san、r1、r2的网络
在san、r1、r2上分别配置
vim/etc/hosts
127.0.0.1san localhost.localdomain
localhost
::1localhost6.localdomain6 localhost6
192.168.33.100san
192.168.33.101 r1
192.168.33.102 r2
192.168.33.201 r1-vip#不要配置在网卡上
192.168.33.202 r2-vip#不要配置在网卡上
10.0.0.1 r1-priv
10.0.0.2 r2-priv
4、在san上配置DNS
yum install bind bind-chroot caching-nameserver
cd /var/named/chroot/etc
cp -p named.caching-nameserver.conf named.conf
vim
/var/named/chroot/etc/named.conf
options {
listen-on port 53 { any; };
listen-on-v6 port 53 { ::1;
};
directory "/var/named";
dump-file
"/var/named/data/cache_dump.db";
statistics-file
"/var/named/data/named_stats.txt";
memstatistics-file
"/var/named/data/named_mem_stats.txt";
allow-query { any; };
allow-query-cache { any; };
};
logging
{
channel default_debug {
file "data/named.run";
severity
dynamic;
};
};
view localhost_resolver {
match-clients { any;
};
match-destinations { any; };
recursion yes;
include
"/etc/named.zones";
};
cp -p named.rfc1912.zones named.zones
vim /var/named/chroot/etc/named.zones
// named.rfc1912.zones:
//
// Provided by Red Hat caching-nameserver
package
//
// ISC BIND named zone configuration for zones recommended
by
// RFC 1912 section 4.1 : localhost TLDs and address zones
//
// See
/usr/share/doc/bind*/sample/ for example named configuration
files.
//
zone "." IN {
type
hint;
file "named.ca";
};
zone "localdomain" IN {
type
master;
file
"localdomain.zone";
allow-update {
none; };
};
zone "localhost" IN {
type
master;
file
"localhost.zone";
allow-update {
none; };
};
zone "0.0.127.in-addr.arpa" IN
{
type
master;
file
"named.local";
allow-update {
none; };
};
zone "0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa"
IN {
type
master;
file
"named.ip6.local";
allow-update {
none; };
};
zone "255.in-addr.arpa" IN {
type master;
file
"named.broadcast";
allow-update {
none; };
};
zone "0.in-addr.arpa" IN {
type
master;
file
"named.zero";
allow-update { none;
};
};
zone "ora11grac.com" IN {
type master;
file
"ora11grac.com.zone";
allow-update { none; };
};
zone "33.168.192.in-addr.arpa" IN {
type
master;
file "33.168.192.local";
allow-update { none; };
};
cd /var/named/chroot/var/named
cp -p named.zero ora11grac.com.zone
cp
-p named.local 33.168.192.local
vim /var/named/chroot/var/named/ora11grac.com.zone
$TTL 86400
@ IN SOA dns.ora11grac.com. root.ora11grac.com. (
42 ;
serial (d. adams)
3H ; refresh
15M ; retry
1W ; expiry
1D ) ;
minimum
@ IN NS dns.ora11grac.com.
r1 IN A 192.168.33.101
r2 IN A
192.168.33.102
r1-vip IN A 192.168.33.201
r2-vip IN A
192.168.33.202
scan IN A 192.168.33.195
scan IN A 192.168.33.196
scan
IN A 192.168.33.197
vim /var/named/chroot/var/named/33.168.192.local
$TTL 86400
@ IN SOA dns.ora11grac.com. root.ora11grac.com.
(
1997022700 ; Serial
28800 ; Refresh
14400 ; Retry
3600000 ;
Expire
86400 ) ; Minimum
@ IN NS dns.ora11grac.com.
101 IN PTR
r1.ora11grac.com.
102 IN PTR r2.ora11grac.com.
201 IN PTR
r1-vip.ora11grac.com.
202 IN PTR r2-vip.ora11grac.com.
195 IN PTR
scan.ora11grac.com.
196 IN PTR scan.ora11grac.com.
197 IN PTR
scan.ora11grac.com.
service named restart
chkconfig named on
完成后在客户端测试
vim /etc/resolve.conf
search ora11grac.com
nameserver 192.168.33.100
然后ping scan.ora11grac.com，或者nslookup scan.ora11grac.com
5、添加用户和组在r1、r2
在r1、r2分别
groupadd -g 501 oinstall
groupadd -g 502 dba
groupadd -g 503
oper
groupadd -g 504 asmadmin
groupadd -g 505 asmdba
groupadd -g 506
asmoper
useradd -u 501 -g oinstall -G dba,oper,asmdba oracle
useradd -u 502 -g oinstall -G dba,asmadmin,asmdba,asmoper grid
在r1、r2上修改配置文件
vim /home/grid/.bashrc
export ORACLE_BASE=/u01/app/oracle
export ORACLE_HOME=/u01/app/11.2.0/grid
export PATH=$ORACLE_HOME/bin:$PATH
export ORACLE_OWNER=oracle
export ORACLE_SID=+ASM1 #r2节点为 export ORACLE_SID=+ASM2
export ORACLE_TERM=vt100
export THREADS_FLAG=native
export
LD_LIBRARY_PATH=$ORACLE_HOME/lib:$LD_LIBRARY_PATH
export
PATH=$ORACLE_HOME/bin:$PATH
export LANG=en_US
alias
sqlplus=‘rlwrap sqlplus‘
alias lsnrctl=‘rlwrap lsnrctl‘
alias
asmcmd=‘rlwrap asmcmd‘
vim /home/oracle/.bashrc
export ORACLE_BASE=/u01/app/oracle
export ORACLE_HOME=$ORACLE_BASE/product/11.2.0/db_1
export PATH=$ORACLE_HOME/bin:$PATH
export ORACLE_OWNER=oracle
export ORACLE_SID=orcl1 #rac2节点为 export ORACLE_SID=orcl2
export ORACLE_TERM=vt100
export THREADS_FLAG=native
export
LD_LIBRARY_PATH=$ORACLE_HOME/lib:$LD_LIBRARY_PATH
export
PATH=$ORACLE_HOME/bin:$PATH
export EDITOR=vi
export
SQLPATH=/home/oracle
export LANG=en_US
alias
sqlplus=‘rlwrap sqlplus‘
alias lsnrctl=‘rlwrap lsnrctl‘
alias rman=‘rlwrap
rman‘
alias dgmgrl=‘rlwrap dgmgrl‘
6、创建目录在r1、r2
在r1、r2分别
mkdir -p /u01/app/11.2.0/grid
chown -R grid:oinstall /u01
mkdir
/u01/app/oracle
chown -R oracle:oinstall /u01/app/oracle
chmod -R
775 /u01/
7、配置grid、oracle用户下的双机互信（需要单步）
在r1上
su - grid
ssh-keygen -t rsa
ssh-keygen -t dsa
cd .ssh
cat *.pub > authorized_keys
在r2上
su - grid
ssh-keygen -t rsa
ssh-keygen -t dsa
cd .ssh
cat *.pub > authorized_keys
在r1上
scp authorized_keys [email protected]:/home/grid/.ssh/k1
在r2上
cat /home/grid/.ssh/k1 >> authorized_keys
scp authorized_keys [email protected]:/home/grid/.ssh/
在r1上
su - oracle
ssh-keygen -t rsa
ssh-keygen -t dsa
cd .ssh
cat *.pub > authorized_keys
在r2上
su - oracle
ssh-keygen -t rsa
ssh-keygen -t dsa
cd .ssh
cat *.pub > authorized_keys
在r1上
scp authorized_keys oracle@r2:/home/oracle/.ssh/k1
在r2上
cat /home/oracle/.ssh/k1 >> authorized_keys
scp authorized_keys oracle@r1:/home/oracle/.ssh/
分别在r1、r2上测试双机信任
ssh r1 date
ssh r2 date
ssh r1-priv date
ssh r2-priv date
8、配置NTP服务器在san上（不在san上做在其它机器上做也可以）
在san上
vim /etc/sysconfig/ntpd
# Drop root to id ‘ntp:ntp‘ by
default.
OPTIONS="-x -u ntp:ntp -p /var/run/ntpd.pid"
# Set to ‘yes‘ to sync hw clock after successful
ntpdate
SYNC_HWCLOCK=no
# Additional options for ntpdate
vim /etc/ntp.conf
restrict default nomodify
restrict
127.0.0.1
server 127.127.1.0
fudge 127.127.1.0 stratum 8
driftfile
/var/lib/ntp/drift
broadcastdelay 0.008
启动ntpd服务
service ntpd restart
chkconfig ntpd on
在客户端r1、r2
ntpdate san
然后使用crontab -e
*/5 * * * * /usr/sbin/ntpdate san
建议拷贝配置文件到r1、r2，然后在r1、r2也启动ntpd，因为grid安装的时候会进行校验
9、配置集中存储san
在san上
yum install scsi-target-utils -y
vim /etc/tgt/targets.conf
<target lun1>
backing-store /dev/sdb1
backing-store /dev/sdc1
backing-store /dev/sdd1
backing-store /dev/sde1
backing-store /dev/sdf1
backing-store /dev/sdf2
backing-store /dev/sdf3
</target>
service tgtd start
tgtadm --lld iscsi -m target --op show
chkconfig
tgtd on
在r1和r2上
yum install iscsi-initiator-utils -y
service iscsid start
iscsiadm -m
discovery -t st -p san
service iscsi start
chkconfig iscsid
on
chkconfig iscsi on
10、配置udev绑定裸设备在r1、r2上
在r1、r2上
ll /dev/sd*
brw-r----- 1 root disk 8,0 Jan4 19:31 /dev/sda
brw-r----- 1
root disk 8,1 Jan4 11:31 /dev/sda1
brw-r----- 1 root disk
8,2 Jan4 19:31 /dev/sda2
brw-r----- 1 root disk 8, 16 Jan4 13:31 /dev/sdb
brw-r----- 1 root disk 8, 17 Jan4 16:30
/dev/sdb1
brw-r----- 1 root disk 8, 32 Jan4 13:31
/dev/sdc
brw-r----- 1 root disk 8, 33 Jan4 16:30
/dev/sdc1
brw-r----- 1 root disk 8, 48 Jan4 13:32
/dev/sdd
brw-r----- 1 root disk 8, 49 Jan4 16:30
/dev/sdd1
brw-r----- 1 root disk 8, 64 Jan4 13:32
/dev/sde
brw-r----- 1 root disk 8, 65 Jan4 16:30
/dev/sde1
brw-r----- 1 root disk 8, 80 Jan4 13:32
/dev/sdf
brw-r----- 1 root disk 8, 81 Jan4 16:29
/dev/sdf1
brw-r----- 1 root disk 8, 96 Jan4 13:33
/dev/sdg
brw-r----- 1 root disk 8, 97 Jan4 16:29
/dev/sdg1
brw-r----- 1 root disk 8, 98 Jan4 16:29 /dev/sdg2
vim /etc/udev/rules.d/60-raw.rules
ACTION=="add", KERNEL=="sdf1", RUN+="/bin/raw /dev/raw/raw1
%N"
ACTION=="add", KERNEL=="sdg1", RUN+="/bin/raw /dev/raw/raw2
%N"
ACTION=="add", KERNEL=="sdg2", RUN+="/bin/raw /dev/raw/raw3
%N"
KERNEL=="raw1", OWNER="grid", GROUP="asmadmin",
MODE="660"
KERNEL=="raw2", OWNER="grid", GROUP="asmadmin",
MODE="660"
KERNEL=="raw3", OWNER="grid", GROUP="asmadmin", MODE="660"
11、安装依赖包
yum install binutils compat-libstdc++ elfutils-libelf elfutils-libelf-devel
gcc gcc-c++ glibc glibc-common glibc-devel glibc-headers ksh libaio libaio-devel
libgcc libstdc++ libstdc++-devel make sysstat unixODBC unixODBC-devel -y
12、安装ASM的RPM包（务必注意版本和平台）
rpm -ivh oracleasm-2.6.18-164.el5-2.0.5-1.el5.x86_64.rpm
oracleasm-support-2.1.7-1.el5.x86_64.rpm
oracleasmlib-2.0.4-1.el5.x86_64.rpm
13、配置ASM
在r1上
/etc/init.d/oracleasm configure
用户指定：grid
组指定：asmadmin
其余都是yes
建立oracleasm磁盘
/etc/init.d/oracleasm createdisk V1 /dev/sdb1
/etc/init.d/oracleasm createdisk V2 /dev/sdc1
/etc/init.d/oracleasm createdisk V3 /dev/sdd1
/etc/init.d/oracleasm createdisk V4 /dev/sde1
在r2上进行扫描
/etc/init.d/oracleasm scandisks
14、用grid用户安装grid校验
./runcluvfy.sh stage -pre crsinst -n r1,r2 -fixup -verbose
15、安装grid
在grid用户下
./runInstaller
在Instalaation Type选择
Install and Configure Grid Infrustructure for a Cluster
在Product Languages选择
Advanced Installation
在Grid Plug and Play Information填写
Cluster Name: r-cluster
SCAN Name: scan.ora11grac.com
SCAN Port: 1521
Configure GNS不勾选
在Cluster Node Information填写
r1.ora11grac.comr1-vip.ora11grac.com
r2.ora11grac.comr2-vip.ora11grac.com
这个vip不要使用ifconfig去添加
在Specify Network Interface Usage对应填写
eth0192.168.33.0Public
eth110.0.0.0Private
在Storage Option Information选择
Automatic Storage Management(ASM)
在Create ASM Disk Group
选择刚刚认出的raw1, raw2, raw3组成Disk Group Name为grid
在Specify ASM Password
填写用户口令
在Failure Isolation Support选择
Do not use Intelligent Platform Management(IPMI)
在Privileged Operating System Groups填写：
ASM Database Administrator (OSDBA) Group: asmdba
ASM Instance Administration Operator (OSOPER) Group: asmoper
ASM Instance Administrator (OSASM) Group：asmadmin
在Specify Installation Location中填写：
Oracle Base: /u01/app/oracle
Software Location: /u01/app/oracle/product/11.2.0/grid
在Create Inventory中填写：
Inventory Directory: /u01/app/oraInventory
oraInventory Group Name: oinstall
安装完成后
在r1运行
/u01/app/oraInventory/orainstRoot.sh
在r2运行
/u01/app/oraInventory/orainstRoot.sh
在r1运行
/u01/app/11.2.0/grid/root.sh
在r2运行
/u01/app/11.2.0/grid/root.sh
16、安装数据库
先要确保所有的重要服务都是ONLINE的
crs_stat -t
一般oc4j会不在线，那就
srvctl enable oc4j
srvctl start oc4j
olsnodes -n
出现r1，r2
这样就OK了
（gsd服务不在线不影响库软件的安装和库的建立）
用grid用户，开启asmca，创建一个ASM磁盘组+ASM
在r1切换到
su - oracle
./runInstaller
在Select Installation Option
选择Install database software only，只安装数据库引擎软件
在Node Selection
勾选r1，r2两个节点
在Select Database Edition选择
Enterprise Edition
在Specify Installation Location填写
Oracle Base: /u01/app/Oracle
Software Location: /u01/app/oracle/product/11.2.0/dbhome_1
执行完毕后运行，在r1、r2运行
/u01/app/oracle/product/11.2.0/db_1/root.sh
完成。