Before rerunning of root.sh script, we must deconfigure it.
[root@gract3 ~]# $GRID_HOME/crs/install/rootcrs.pl -deconfig
Using configuration parameter file: /u01/app/12102/grid/crs/install/crsconfig_params2014/08/16 07:31:41
CLSRSC-38: Oracle Clusterware stack is not active on this node2014/08/16 07:31:41 CLSRSC-311:
Failed to verify resourcesDied at /u01/app/12102/grid/crs/install/crsdeconfig.pm line 333.
Use -force switch as CRS is not up
[root@gract3 ~]# $GRID_HOME/crs/install/rootcrs.pl -deconfig -force
Configuration
- Upgrade the flex cluster and db to 12.1.0.2 smoothly.
- Delete one hub node(gract3) from the clusterware.
- Add the node back, but it was failed running root.sh with CLSRSC-366
Review traces from failed running of root.sh – CLSRSC-366 error
Locate any files with CLSRSC-366 error
[grid@gract3 grid]$ find $GRID_HOME/cfgtoollogs/crsconfig -exec grep -l CLSRSC-366 {} \;
/u01/app/12102/grid/cfgtoollogs/crsconfig/rootcrs_gract3_2014-08-16_07-22-08AM.log
Find error details
[grid@gract3 grid]$ find $GRID_HOME/cfgtoollogs/crsconfig -exec grep CLSRSC-366 {} \;
> CLSRSC-366: Failed to import credentials for ASM
> CLSRSC-366: Failed to import credentials for ASM
2014-08-16 07:24:16: CLSRSC-366: Failed to import credentials for ASM
Check trace file more in detail:
2014-08-16 07:24:15: s_run_as_user2: Running /bin/su grid -c ' echo CLSRSC_START;
/u01/app/12102/grid/bin/kfod op=credimport wrap=/u01/app/12102/grid/gpnp/seed/asm/credentials.xml olr=TRUE force=TRUE '
...
2014-08-16 07:24:16: kfod op=credimport rc: 139
2014-08-16 07:24:16: Failed to enable flex ASM on local node, error: bash: line 1: 27788 Segmentation fault (core dumped)
/u01/app/12102/grid/bin/kfod op=credimport wrap=/u01/app/12102/grid/gpnp/seed/asm/credentials.xml olr=TRUE force=TRUE
2014-08-16 07:24:16: Executing cmd: /u01/app/12102/grid/bin/clsecho -p has -f clsrsc -m 366
2014-08-16 07:24:16: Command output:
> CLSRSC-366: Failed to import credentials for ASM
--> kfod dump core :
Find related traces:
[root@gract3 log]# find $GRID_HOME/log -type f -printf "%CY-%Cm-%Cd %CH:%CM:%CS %h/%f\n" | sort -n | more
2014-08-16 07:24:15.6642347810 /u01/app/12102/grid/log/gract3/client/GIPC_27788.log
2014-08-16 07:24:15.7582377850 /u01/app/12102/grid/log/diag/clients/user_grid/host_1786982425_82/lck/AM_1762783_4031814035.lck
2014-08-16 07:24:15.7592378170 /u01/app/12102/grid/log/diag/clients/user_grid/host_1786982425_82/metadata/INCIDENT.ams
2014-08-16 07:24:15.7652380080 /u01/app/12102/grid/log/diag/clients/user_grid/host_1786982425_82/trace/ora_27788_140265833375296.trc
2014-08-16 07:24:15.7652380080 /u01/app/12102/grid/log/diag/clients/user_grid/host_1786982425_82/trace/ora_27788_140265833375296.trm
2014-08-16 07:24:15.7662380400 /u01/app/12102/grid/log/diag/clients/user_grid/host_1786982425_82/alert/log.xml
2014-08-16 07:24:15.7662380400 /u01/app/12102/grid/log/diag/clients/user_grid/host_1786982425_82/sweep/stg1_9_inc.swp
2014-08-16 07:24:15.7662380400 /u01/app/12102/grid/log/diag/clients/user_grid/host_1786982425_82/trace/sqlnet.log
2014-08-16 07:24:15.7712381990 /u01/app/12102/grid/log/diag/clients/user_grid/host_1786982425_82/incident/incdir_9/ora_27788_140265833375296_i9.trc
2014-08-16 07:24:15.7712381990 /u01/app/12102/grid/log/diag/clients/user_grid/host_1786982425_82/incident/incdir_9/ora_27788_140265833375296_i9.trm
2014-08-16 07:24:15.7712381990 /u01/app/12102/grid/log/diag/clients/user_grid/host_1786982425_82/sweep/stg1_9_inc2.swp
root@gract3 log]# cat /u01/app/12102/grid/log/diag/clients/user_grid/host_1786982425_82/trace/ora_27788_140265833375296.trc
Trace file /u01/app/12102/grid/log/diag/clients/user_grid/host_1786982425_82/trace/ora_27788_140265833375296.trc
DDE: Flood control is not active
Incident 9 created, dump file: /u01/app/12102/grid/log/diag/clients/user_grid/host_1786982425_82/incident/incdir_9/ora_27788_140265833375296_i9.trc
oci-24550 [11] [[si_signo=11] [si_errno=0] [si_code=1] [si_int=15332088] [si_ptr=0xe9f2f8] [si_addr=(nil)]] [] [] [] [] [] [] [] [] [] []
[root@gract3 log]# cat /u01/app/12102/grid/log/diag/clients/user_grid/host_1786982425_82/incident/incdir_9/ora_27788_140265833375296_i9.trc
Dump file /u01/app/12102/grid/log/diag/clients/user_grid/host_1786982425_82/incident/incdir_9/ora_27788_140265833375296_i9.trc
[TOC00000]
Jump to table of contents
Dump continued from file: /u01/app/12102/grid/log/diag/clients/user_grid/host_1786982425_82/trace/ora_27788_140265833375296.trc
[TOC00001]
oci-24550 [11] [[si_signo=11] [si_errno=0] [si_code=1] [si_int=15332088] [si_ptr=0xe9f2f8] [si_addr=(nil)]] [] [] [] [] [] [] [] [] [] []
[TOC00001-END]
[TOC00002]
========= Dump for incident 9 (oci 24550 [11]) ========
Tracing is in restricted mode!
[TOC00003]
----- Short Call Stack Trace -----
dbgexPhaseII()+878<-dbgexProcessError()+2539<-dbgeExecuteForError()+110<-dbgePostErrorDirect()+2230<-kpeDbgSignalHandler()
+302<-skgesig_sigactionHandler()+229<-__sighandler()<-kgfz_importCluster()+2685<-kfodCredImport()+173<-kfodExecute()+1924
<-kfod_main()+899<-lpmcall()+680<-lpmpmai()+420<-main()+458<-__libc_start_main()+253[TOC00003-END]
-->kfod executable crashes with following stack
kgfz_importCluster()
kfodCredImport()
kfodExecute()
Bug Reference
- Bug 18406774 : CLSXWRAP_WRAPSECTGETPROP() RETURNS SUCCESS EVEN WHEN PROPERTY IS MISSING
- Bug 19234452 : LNX64-12C-UD:FLEX CLUSTER ADD NODE, FAILED TO IMPORT CREDENTIALS FOR ASM
Stack trace from Bug 19234452
(gdb) where
#0 0x0000003e4d278900 in strlen () from /lib64/libc.so.6
#1 0x00002aaaafffe3ce in kgfz_importCluster (ctxt=0x6f3d90,
wfile=0x639814 "/u01/app/12.1.0/grid_1/gpnp/seed/asm/credentials.xml",
flags=1) at kgfz.c:3316
#2 0x0000000000407eed in kfodCredImport
#3 0x0000000000408c84 in kfodExecute
#4 0x0000000000406ea3 in kfod_main
Bug 18406774 - Basebug of BUG 19234452 - is fixed by upcomming October Patch Bundle
--> Will test asap !