Startup and check CRS and verify your disk groups
# $GRID_HOME/bin/crsctl start crs
CRS-4123: Oracle High Availability Services has been started.
# $GRID_HOME/bin/crsctl check crs
CRS-4638: Oracle High Availability Services is online
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4530: Communications failure contacting Cluster Synchronization Services daemon
CRS-4534: Cannot communicate with Event Manager
$ asmcmd lsdg
Connected to an idle instance.
ASMCMD-8102: no connection to Oracle ASM; command requires Oracle ASM to run
--> From GRID alert.log
[cssd(10157)]CRS-1714:Unable to discover any voting files, retrying discovery in 15 seconds;
Details at (:CSSNM00070:) in /u01/app/11204/grid/log/grac42/cssd/ocssd.log
$ /etc/init.d/oracleasm listdisks
--> No ASM disks configured - Installation uses udev devices!
Get your disk discovery string and verify your /dev/asm directory
$GRID_HOME/bin/gpnptool get
DiscoveryString="/dev/asm*
SPFile="+DATA/grac4/asmparameterfile/registry.253.825944079"
$ ls -l /dev/asm*
brw-rw----. 1 grid asmadmin 252, 5 Mar 18 13:42 /dev/asmdisk_OF-disk1
brw-rw----. 1 grid asmadmin 252, 6 Mar 18 13:42 /dev/asmdisk_OF-disk2
brw-rw----. 1 grid asmadmin 252, 7 Mar 18 13:42 /dev/asmdisk_OF-disk3
$ kfed read /dev/asmdisk_OF-disk1 | egrep 'dskname|grpname'
kfdhdb.dskname: OPENFILER_DG_0000 ; 0x028: length=17
kfdhdb.grpname: OPENFILER_DG ; 0x048: length=12
[grid@grac42 grac42]$ kfed read /dev/asmdisk_OF-disk2 | egrep 'dskname|grpname'
kfdhdb.dskname: OPENFILER_DG_0001 ; 0x028: length=17
kfdhdb.grpname: OPENFILER_DG ; 0x048: length=12
[grid@grac42 grac42]$ kfed read /dev/asmdisk_OF-disk3 | egrep 'dskname|grpname'
kfdhdb.dskname: OPENFILER_DG_0002 ; 0x028: length=17
kfdhdb.grpname: OPENFILER_DG ; 0x048: length=12
Verify voting file location locally
# more /etc/oracle/ocr.loc
ocrconfig_loc=+OCR
local_only=false
Verify voting file location on a working instance
$ crsctl query css votedisk
## STATE File Universal Id File Name Disk group
-- ----- ----------------- --------- ---------
1. ONLINE b0e94e5d83054fe9bf58b6b98bfacd65 (/dev/asmdisk5_udev_sdf1) [OCR]
2. ONLINE 88c2a08b4c8c4f85bf0109e0990388e4 (/dev/asmdisk6_udev_sdg1) [OCR]
3. ONLINE 1108f9a41e814fb2bfed879ff0039dd0 (/dev/asmdisk7_udev_sdh1) [OCR]
Check ASM devices
ON Working Node
[root@grac41 rules.d]# ls /dev/asm*
/dev/asmdisk10_ssd3 /dev/asmdisk2_test /dev/asmdisk4_udev_sde1 /dev/asmdisk8_ssd1 /dev/asmdisk_OF-disk3
/dev/asmdisk1_fra /dev/asmdisk2_udev_sdc1 /dev/asmdisk5_udev_sdf1 /dev/asmdisk9_ssd2
/dev/asmdisk1_test /dev/asmdisk3_test /dev/asmdisk6_udev_sdg1 /dev/asmdisk_OF-disk1
/dev/asmdisk1_udev_sdb1 /dev/asmdisk3_udev_sdd1 /dev/asmdisk7_udev_sdh1 /dev/asmdisk_OF-disk2
Failing Node
grid@grac42 grac42]$ ls -l /dev/asm*
brw-rw----. 1 grid asmadmin 252, 5 Mar 18 13:42 /dev/asmdisk_OF-disk1
brw-rw----. 1 grid asmadmin 252, 6 Mar 18 13:42 /dev/asmdisk_OF-disk2
brw-rw----. 1 grid asmadmin 252, 7 Mar 18 13:42 /dev/asmdisk_OF-disk3
--> All disks for Diskgroup OCR are missing from /dev/asm*
Solution : Fix udev file and verify that our OCR diskgroup can be mounted
$ udevadm control --reload-rules
$ start_udev
Verify ASM disks:
[root@grac43 rules.d]# $GRID_HOME/bin/kfed read /dev/asmdisk1_udev_sdf1| egrep 'dskname|grpname'
kfdhdb.dskname: OCR_0000 ; 0x028: length=8
kfdhdb.grpname: OCR ; 0x048: length=3
[root@grac43 rules.d]# $GRID_HOME/bin/kfed read /dev/asmdisk1_udev_sdg1| egrep 'dskname|grpname'
kfdhdb.dskname: OCR_0001 ; 0x028: length=8
kfdhdb.grpname: OCR ; 0x048: length=3
[root@grac43 rules.d]# $GRID_HOME/bin/kfed read /dev/asmdisk1_udev_sdh1| egrep 'dskname|grpname'
kfdhdb.dskname: OCR_0002 ; 0x028: length=8
kfdhdb.grpname: OCR ; 0x048: length=3
Restart CRS ( as root ) and verify ASM status
# $GRID_HOME/bin/crsctl stop crs -f
# $GRID_HOME/bin/crsctl start crs
$ $GRID_HOME/bin/asmcmd lsdg
State Type Rebal Sector Block AU Total_MB Free_MB Req_mir_free_MB Usable_file_MB Offline_disks Voting_files Name
MOUNTED NORMAL N 512 4096 1048576 40944 20431 10236 5097 0 N DATA/
MOUNTED EXTERN N 512 4096 1048576 204797 173814 0 173814 0 N FRA/
MOUNTED NORMAL N 512 4096 1048576 6141 5091 2047 1522 0 Y OCR/
MOUNTED EXTERN N 512 4096 1048576 5739 4565 0 4565 0 N OPENFILER_DG/
MOUNTED NORMAL N 512 4096 1048576 3057 1995 1019 488 0 N SSD/
# $GRID_HOME/bin/crsctl start crs
CRS-4123: Oracle High Availability Services has been started.
# $GRID_HOME/bin/crsctl check crs
CRS-4638: Oracle High Availability Services is online
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4530: Communications failure contacting Cluster Synchronization Services daemon
CRS-4534: Cannot communicate with Event Manager
$ asmcmd lsdg
Connected to an idle instance.
ASMCMD-8102: no connection to Oracle ASM; command requires Oracle ASM to run
--> From GRID alert.log
[cssd(10157)]CRS-1714:Unable to discover any voting files, retrying discovery in 15 seconds;
Details at (:CSSNM00070:) in /u01/app/11204/grid/log/grac42/cssd/ocssd.log
$ /etc/init.d/oracleasm listdisks
--> No ASM disks configured - Installation uses udev devices!
Get your disk discovery string and verify your /dev/asm directory
$GRID_HOME/bin/gpnptool get
DiscoveryString="/dev/asm*
SPFile="+DATA/grac4/asmparameterfile/registry.253.825944079"
$ ls -l /dev/asm*
brw-rw----. 1 grid asmadmin 252, 5 Mar 18 13:42 /dev/asmdisk_OF-disk1
brw-rw----. 1 grid asmadmin 252, 6 Mar 18 13:42 /dev/asmdisk_OF-disk2
brw-rw----. 1 grid asmadmin 252, 7 Mar 18 13:42 /dev/asmdisk_OF-disk3
$ kfed read /dev/asmdisk_OF-disk1 | egrep 'dskname|grpname'
kfdhdb.dskname: OPENFILER_DG_0000 ; 0x028: length=17
kfdhdb.grpname: OPENFILER_DG ; 0x048: length=12
[grid@grac42 grac42]$ kfed read /dev/asmdisk_OF-disk2 | egrep 'dskname|grpname'
kfdhdb.dskname: OPENFILER_DG_0001 ; 0x028: length=17
kfdhdb.grpname: OPENFILER_DG ; 0x048: length=12
[grid@grac42 grac42]$ kfed read /dev/asmdisk_OF-disk3 | egrep 'dskname|grpname'
kfdhdb.dskname: OPENFILER_DG_0002 ; 0x028: length=17
kfdhdb.grpname: OPENFILER_DG ; 0x048: length=12
Verify voting file location locally
# more /etc/oracle/ocr.loc
ocrconfig_loc=+OCR
local_only=false
Verify voting file location on a working instance
$ crsctl query css votedisk
## STATE File Universal Id File Name Disk group
-- ----- ----------------- --------- ---------
1. ONLINE b0e94e5d83054fe9bf58b6b98bfacd65 (/dev/asmdisk5_udev_sdf1) [OCR]
2. ONLINE 88c2a08b4c8c4f85bf0109e0990388e4 (/dev/asmdisk6_udev_sdg1) [OCR]
3. ONLINE 1108f9a41e814fb2bfed879ff0039dd0 (/dev/asmdisk7_udev_sdh1) [OCR]
Check ASM devices
ON Working Node
[root@grac41 rules.d]# ls /dev/asm*
/dev/asmdisk10_ssd3 /dev/asmdisk2_test /dev/asmdisk4_udev_sde1 /dev/asmdisk8_ssd1 /dev/asmdisk_OF-disk3
/dev/asmdisk1_fra /dev/asmdisk2_udev_sdc1 /dev/asmdisk5_udev_sdf1 /dev/asmdisk9_ssd2
/dev/asmdisk1_test /dev/asmdisk3_test /dev/asmdisk6_udev_sdg1 /dev/asmdisk_OF-disk1
/dev/asmdisk1_udev_sdb1 /dev/asmdisk3_udev_sdd1 /dev/asmdisk7_udev_sdh1 /dev/asmdisk_OF-disk2
Failing Node
grid@grac42 grac42]$ ls -l /dev/asm*
brw-rw----. 1 grid asmadmin 252, 5 Mar 18 13:42 /dev/asmdisk_OF-disk1
brw-rw----. 1 grid asmadmin 252, 6 Mar 18 13:42 /dev/asmdisk_OF-disk2
brw-rw----. 1 grid asmadmin 252, 7 Mar 18 13:42 /dev/asmdisk_OF-disk3
--> All disks for Diskgroup OCR are missing from /dev/asm*
Solution : Fix udev file and verify that our OCR diskgroup can be mounted
$ udevadm control --reload-rules
$ start_udev
Verify ASM disks:
[root@grac43 rules.d]# $GRID_HOME/bin/kfed read /dev/asmdisk1_udev_sdf1| egrep 'dskname|grpname'
kfdhdb.dskname: OCR_0000 ; 0x028: length=8
kfdhdb.grpname: OCR ; 0x048: length=3
[root@grac43 rules.d]# $GRID_HOME/bin/kfed read /dev/asmdisk1_udev_sdg1| egrep 'dskname|grpname'
kfdhdb.dskname: OCR_0001 ; 0x028: length=8
kfdhdb.grpname: OCR ; 0x048: length=3
[root@grac43 rules.d]# $GRID_HOME/bin/kfed read /dev/asmdisk1_udev_sdh1| egrep 'dskname|grpname'
kfdhdb.dskname: OCR_0002 ; 0x028: length=8
kfdhdb.grpname: OCR ; 0x048: length=3
Restart CRS ( as root ) and verify ASM status
# $GRID_HOME/bin/crsctl stop crs -f
# $GRID_HOME/bin/crsctl start crs
$ $GRID_HOME/bin/asmcmd lsdg
State Type Rebal Sector Block AU Total_MB Free_MB Req_mir_free_MB Usable_file_MB Offline_disks Voting_files Name
MOUNTED NORMAL N 512 4096 1048576 40944 20431 10236 5097 0 N DATA/
MOUNTED EXTERN N 512 4096 1048576 204797 173814 0 173814 0 N FRA/
MOUNTED NORMAL N 512 4096 1048576 6141 5091 2047 1522 0 Y OCR/
MOUNTED EXTERN N 512 4096 1048576 5739 4565 0 4565 0 N OPENFILER_DG/
MOUNTED NORMAL N 512 4096 1048576 3057 1995 1019 488 0 N SSD/
No comments:
Post a Comment