Category: ASM Generic

Check ASM disk status and fix problems

Check ASM disk status

Check ASM disk status 

SQL> select    g.name disk_grp, o.group_number, operation , est_minutes from gv$asm_operation o,    gv$asm_diskgroup g
         where g.group_number = o.group_number and o.inst_id = g.inst_id;

no rows selected --> No rebalance operation in progress 

SQL> select d.inst_id, dg.name dg_name,  dg.state dg_state,    dg.type,d.name, d.DISK_NUMBER dsk_no, d.MOUNT_STATUS, d.HEADER_STATUS, d.MODE_STATUS,
             d.STATE, d. PATH, d.FAILGROUP  FROM GV$ASM_DISK d,  gv$asm_diskgroup dg
          where dg.group_number(+)=d.group_number and d.inst_id = dg.inst_id order by d.inst_id, d.group_number;
   INST_ID DG_NAME    DG_STATE     TYPE    NAME              DSK_NO MOUNT_S HEADER_STATU MODE_ST STATE    PATH                     FAILGROUP
---------- ---------- ---------- ------ -------------------- ------- ------- ------------ ------- -------- ---------------------------------------- ---------------
     1 OCR          MOUNTED     NORMAL OCR_0000           0 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk1_udev_sdf1            OCR_0000
     1 OCR          MOUNTED     NORMAL OCR_0002           2 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk1_udev_sdh1            OCR_0002
     1 OCR          MOUNTED     NORMAL OCR_0001           1 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk1_udev_sdg1            OCR_0001
     1 FRA          MOUNTED     EXTERN FRA_0002           2 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk_fra2                FRA_0002
     1 FRA          MOUNTED     EXTERN FRA_0001           1 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk_fra1                FRA_0001
     1 DATA       MOUNTED     NORMAL DATA_0001           1 CACHED  CANDIDATE      ONLINE  NORMAL   ORCL:DATA2                    DATA_0001
     1 DATA       MOUNTED     NORMAL DATA_0002           2 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk1_udev_sdd1            DATA_0002
     1 DATA       MOUNTED     NORMAL DATA_0003           3 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk1_udev_sde1            DATA_0003
     1 SSD          MOUNTED     NORMAL SSD_0002           2 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk8_ssd3                SSD_0002
     1 SSD          MOUNTED     NORMAL SSD_0000           0 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk8_ssd1                SSD_0000
     1 SSD          MOUNTED     NORMAL SSD_0001           1 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk8_ssd2                SSD_0001
     2 OCR          MOUNTED     NORMAL OCR_0002           2 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk1_udev_sdh1            OCR_0002
     2 OCR          MOUNTED     NORMAL OCR_0000           0 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk1_udev_sdf1            OCR_0000
     2 OCR          MOUNTED     NORMAL OCR_0001           1 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk1_udev_sdg1            OCR_0001
     2 FRA          MOUNTED     EXTERN FRA_0002           2 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk_fra2                FRA_0002
     2 FRA          MOUNTED     EXTERN FRA_0001           1 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk_fra1                FRA_0001
     2 DATA       MOUNTED     NORMAL DATA_0001           1 CACHED  CANDIDATE      ONLINE  NORMAL   /dev/asmdisk1_udev_sdc1            DATA_0001
     2 DATA       MOUNTED     NORMAL DATA_0003           3 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk1_udev_sde1            DATA_0003
     2 DATA       MOUNTED     NORMAL DATA_0002           2 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk1_udev_sdd1            DATA_0002
     2 SSD          MOUNTED     NORMAL SSD_0002           2 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk8_ssd3                SSD_0002
     2 SSD          MOUNTED     NORMAL SSD_0001           1 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk8_ssd2                SSD_0001
     2 SSD          MOUNTED     NORMAL SSD_0000           0 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk8_ssd1                SSD_0000
     2 ASMLIB_DG  MOUNTED     NORMAL ASMLIB_DG_0002           2 CACHED  CANDIDATE      ONLINE  NORMAL   /dev/oracleasm/disks/ASMLIB_DISK3        ASMLIB_DG_0002
     2 ASMLIB_DG  MOUNTED     NORMAL ASMLIB_DG_0001           1 CACHED  CANDIDATE      ONLINE  NORMAL   /dev/oracleasm/disks/ASMLIB_DISK2        ASMLIB_DG_0001
     2 ASMLIB_DG  MOUNTED     NORMAL ASMLIB_DG_0000           0 CACHED  CANDIDATE      ONLINE  NORMAL   /dev/oracleasm/disks/ASMLIB_DISK1        ASMLIB_DG_0000
     3 OCR          MOUNTED     NORMAL OCR_0001           1 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk1_udev_sdg1            OCR_0001
     3 OCR          MOUNTED     NORMAL OCR_0002           2 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk1_udev_sdh1            OCR_0002
     3 OCR          MOUNTED     NORMAL OCR_0000           0 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk1_udev_sdf1            OCR_0000
     3 FRA          MOUNTED     EXTERN FRA_0001           1 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk_fra1                FRA_0001
     3 FRA          MOUNTED     EXTERN FRA_0002           2 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk_fra2                FRA_0002
     3 DATA       MOUNTED     NORMAL DATA_0003           3 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk1_udev_sde1            DATA_0003
     3 DATA       MOUNTED     NORMAL DATA_0001           1 CACHED  CANDIDATE      ONLINE  NORMAL   /dev/asmdisk1_udev_sdc1            DATA_0001
     3 DATA       MOUNTED     NORMAL DATA_0002           2 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk1_udev_sdd1            DATA_0002
     3 SSD          MOUNTED     NORMAL SSD_0002           2 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk8_ssd3                SSD_0002
     3 SSD          MOUNTED     NORMAL SSD_0000           0 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk8_ssd1                SSD_0000
     3 SSD          MOUNTED     NORMAL SSD_0001           1 CACHED  MEMBER      ONLINE  NORMAL   /dev/asmdisk8_ssd2                SSD_0001
--> Here we have some problems 
    1. ASMLIB_DG is only mounted on Node 2 - disks status CANDIDATE 
    2. Disk DATA_0001 is handled by ASMLIB at Node 1 but is using UDEV at Node2 , Node 3
Pages: Page 1, Page 2, Page 3

ORA-15040, ORA-15042 errors mounting a diskgroup

Check current status

Try manually mount diskgroup
ASMCMD> mount -a
ORA-15032: not all alterations performed
ORA-15040: diskgroup is incomplete
ORA-15042: ASM disk "1" is missing from group number "1" 

Check ASM alert.log
SQL> alter diskgroup ACFS mount 
NOTE: cache registered group ACFS number=1 incarn=0x09884abc
NOTE: cache began mount (first) of group ACFS number=1 incarn=0x09884abc
NOTE: Assigning number (1,0) to disk (/dev/oracleasm/disks/ACFS_DATA)
Mon Aug 19 10:29:24 2013
NOTE: GMON heartbeating for grp 1
GMON querying group 1 at 334 for pid 31, osid 8882
NOTE: Assigning number (1,1) to disk ()
GMON querying group 1 at 335 for pid 31, osid 8882
NOTE: cache dismounting (clean) group 1/0x09884ABC (ACFS) 
NOTE: messaging CKPT to quiesce pins Unix process pid: 8882, image: oracle@grac1.example.com (TNS V1-V3)
NOTE: dbwr not being msg'd to dismount
NOTE: lgwr not being msg'd to dismount
NOTE: cache dismounted group 1/0x09884ABC (ACFS) 
NOTE: cache ending mount (fail) of group ACFS number=1 incarn=0x09884abc
NOTE: cache deleting context for group ACFS 1/0x09884abc
GMON dismounting group 1 at 336 for pid 31, osid 8882
NOTE: Disk  in mode 0x8 marked for de-assignment
NOTE: Disk  in mode 0x8 marked for de-assignment
ERROR: diskgroup ACFS was not mounted
ORA-15032: not all alterations performed
ORA-15040: diskgroup is incomplete
ORA-15042: ASM disk "1" is missing from group number "1" 
ERROR: alter diskgroup ACFS mount
Mon Aug 19 10:29:27 2013
ASM Health Checker found 1 new failures

Display even dismounted diskgroups
$  asmcmd lsdg --discovery
State       Type    Rebal  Sector  Block       AU  Total_MB  Free_MB  Req_mir_free_MB  Usable_file_MB  Offline_disks  Voting_files  Name
DISMOUNTED          N           0   4096        0         0        0                0               0              0             N  ACFS/
MOUNTED     NORMAL  N         512   4096  1048576     15342     9932             5114            2409              0             N  DATA/
MOUNTED     NORMAL  N         512   4096  1048576      6141     5217             2047            1585              0             Y  OCR/
--> diskgroup ACFS is still dismounted  
Try to mount the ASM diskgroup with force option and check the available disks
As we have only a single disk available for NORMAL redundancy the mount fails and we need to use the force option  
SQL> alter diskgroup ACFS mount force;
Diskgroup altered.

Verify the disk status after mount force command. 
$ asmcmd lsdsk -p -G ACFS
Group_Num  Disk_Num      Incarn  Mount_Stat  Header_Stat  Mode_Stat  State   Path
        1         1  3915954601  MISSING     UNKNOWN      OFFLINE    NORMAL  
        1         0  3915954600  CACHED      MEMBER       ONLINE     NORMAL  /dev/oracleasm/disks/ACFS_DATA
$ asmcmd lsdg
State    Type    Rebal  Sector  Block       AU  Total_MB  Free_MB  Req_mir_free_MB  Usable_file_MB  Offline_disks  Voting_files  Name
MOUNTED  NORMAL  N         512   4096  1048576      1019      122                0              61              1             N  ACFS/
MOUNTED  NORMAL  N         512   4096  1048576     15342     9932             5114            2409              0             N  DATA/
MOUNTED  NORMAL  N         512   4096  1048576      6141     5217             2047            1585              0             Y  OCR/
           1585              0             Y  OCR/
Try to read and verify ASM disk header ( on all instances )
# $GRID_HOME/bin/kfed read  /dev/sdj1
kfbh.endian:                          0 ; 0x000: 0x00
kfbh.hard:                            0 ; 0x001: 0x00
kfbh.type:                            0 ; 0x002: KFBTYP_INVALID
kfbh.datfmt:                          0 ; 0x003: 0x00
kfbh.block.blk:                       0 ; 0x004: blk=0
kfbh.block.obj:                       0 ; 0x008: file=0
kfbh.check:                           0 ; 0x00c: 0x00000000
kfbh.fcn.base:                        0 ; 0x010: 0x00000000
kfbh.fcn.wrap:                        0 ; 0x014: 0x00000000
kfbh.spare1:                          0 ; 0x018: 0x00000000
kfbh.spare2:                          0 ; 0x01c: 0x00000000
7FB74EC83400 00000000 00000000 00000000 00000000  [................]
        Repeat 63 times
7FB74EC83800 0000FF00 0003FBB8 000032FC 0003DA6D  [.........2..m...]
7FB74EC83810 0000FEF5 00000000 00000002 00000002  [................]
7FB74EC83940 00000000 00000000 00000000 01000000  [................]
7FB74EC83950 00000000 00000000 00000000 001C001C  [................]
7FB74EC83960 00000001 00000000 00000000 00000000  [................]
7FB74EC83970 00000000 00000004 00008196 00000000  [................]
7FB74EC83980 00000000 00000000 00000000 00000000  [................]
  Repeat 167 times
KFED-00322: file not found; arguments: [kfbtTraverseBlock] [Invalid OSM block type] [] [0]
--> This is not a valid  ASM header - let's create new ASM disk:
Pages: Page 1, Page 2, Page 3