#!/bin/bash
VER=2.0.1
MODATE="2011 Feb 14"

# todo
# 1. put cluster ldap query in it's own file.
# 2. read cn location from file's dn:
# 3. check missing object cases, like volume and ncpserver together

##############################################################################
#  ncsvr - Novell Cluster Service Volume Resource Plugin for Supportconfig
#  Copyright (C) 2011 Novell, Inc.
#
#  This plugin performs an authenticated, non-destructive LDAP search of
#  the eDirectory database for Novell Cluster Services volume resources. 
#  Information about each eDirectory object associated with the volume 
#  resource is used to help confirm the proper object association. It also 
#  checks for expected attributes on each of the objects.
#
##############################################################################
#
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License
#  along with this program; if not, write to the Free Software
#  Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
#
#  Authors/Contributors:
#     Jason Record (jrecord@novell.com)
#
##############################################################################

# global variables
##############################################################################
if [ -x /usr/ldaptools/bin/ldapsearch ]; then
	LDAP_BIN=/usr/ldaptools/bin/ldapsearch
elif [ -x /opt/novell/eDirectory/bin/ldapsearch ]; then
	LDAP_BIN=/opt/novell/eDirectory/bin/ldapsearch 
elif [ -x /usr/bin/ldapsearch ]; then
	LDAP_BIN=/usr/bin/ldapsearch
else
	LDAP_BIN="" 
fi
DATESTAMP=$(date +%y%m%d)
TIMESTAMP=$(date +%H%M%S)
HOSTNAME=$(hostname)
test -z "$HOSTNAME" && HOSTNAME=unknown
# LOG is set by supportconfig
LOGROOT="${LOG:="/var/log/nts_ncsvr_${HOSTNAME}_${DATESTAMP}_${TIMESTAMP}"}/ldap-ncsvr-files"
FILE_CLUSTERS="${LOGROOT}/clusters.txt"
LOG_SUMMARY="${LOGROOT}/Analysis.txt"
TOTAL_ERRORS=0
TOTAL_OBJ=0
TOTAL_ATTR=0
TOTAL_LINK=0
TOTAL_STAT=0
TOTAL_VOLRES=0
TOTAL_VOLRES_ERRORS=0
PATH=/bin:/sbin:$PATH
export PATH
FMT_INTRO="%-20s %s\n"
FMT_OBJ="%-35s %s\n"
FMT_SHORT="%-9s %s\n"
FMT_LONG="%-45s %s\n"
FMT_QUERY="%-22s %s\n"
FMT_CURRENT=$FMT_INTRO

# function definitions
##############################################################################

title() {
	log_detail "---------------------------------------------------------"
	log_detail " Novell Open Enterprise Server"
	log_detail " NCS Volume Resource Plugin for Supportconfig"
	log_detail " Version: $VER ($MODATE)"
	log_detail " Date:    $(date +"%D %T")"
	log_detail "---------------------------------------------------------"
}

get_credentials() {
	KEYFOUND=1
	DERFILE=".$(namconfig get preferred-server 2>/dev/null | cut -d= -f2).der"
	CERT_DIRECTORIES="/var/lib/novell-lum /var/nam"
	for CERT_DIR in $CERT_DIRECTORIES
	do
		AUTHKEY="${CERT_DIR}/${DERFILE}"
		if [ -s $AUTHKEY ]; then
			log_summary "Using" "$AUTHKEY"
			KEYFOUND=1
			break
		else
			log_summary "File not found" "$AUTHKEY"
			KEYFOUND=0
		fi
	done

	if [ $KEYFOUND = 0 ]; then
		log_detail "ERROR: Missing *.der file for TLS authentication in $CERT_DIRECTORIES"
		log_detail "       Consider creating a certificate with namconfig -k"
		log_detail
		((TOTAL_ERRORS++))
		show_summary
		exit 1
	fi

	if [ -x $(which namconfig 2>/dev/null) ]; then
		PREF_SERVER=$(namconfig get preferred-server | cut -d= -f2)
		test -z "$PREF_SERVER" && PREF_SERVER=localhost
	else
		PREF_SERVER=localhost
	fi
	
	if [ -z "$NCSVR_ADMIN" -a -z "$NCSVR_PASS" ]; then
		if [ -x /opt/novell/ncs/bin/ncs-casautil ]; then
			log_summary "Credentials" "CASA"
			NCSVR_ADMIN=$(SECRET_ID=NovellClusterServices.Novell /opt/novell/ncs/bin/ncs-casautil -d | sed -e '1d' | head -1)
			NCSVR_PASS=$(SECRET_ID=NovellClusterServices.Novell /opt/novell/ncs/bin/ncs-casautil -d | sed -e '1d' | tail -1)
		else
			log_detail "ERROR: Unable to access cluster credentials"
			log_detail "       Consider setting NCSVR_ADMIN and NCSVR_PASS in the environment first."
			log_detail "       For example, run:"
			log_detail "       NCSVR_ADMIN=\"cn=admin,o=novell\" NCSVR_PASS=\"Novell\" supportconfig"
			log_detail
			exit 2
		fi
	else
		log_summary "Credentials" "Environment, NCSVR_ADMIN and NCSVR_PASS"
	fi
	if [ -z "$NCSVR_ADMIN" -a -n "$NCSVR_PASS" ]; then
		log_detail "ERROR: NCSVR_ADMIN is undefined"
		log_detail "       Define both NCSVR_ADMIN and NCSVR_PASS"
		log_detail
		exit 4
	elif [ -n "$NCSVR_ADMIN" -a -z "$NCSVR_PASS" ]; then
		log_detail "ERROR: NCSVR_PASS is undefined"
		log_detail "       Define both NCSVR_ADMIN and NCSVR_PASS"
		log_detail
		exit 6
	elif [ -z "$NCSVR_ADMIN" -a -z "$NCSVR_PASS" ]; then
		log_detail "ERROR: NCSVR_ADMIN and NCSVR_PASS are undefined"
		log_detail "       Define both NCSVR_ADMIN and NCSVR_PASS"
		log_detail
		exit 8
	fi

	AUTH_OPTIONS="-e $AUTHKEY -D $NCSVR_ADMIN -w $NCSVR_PASS -h $PREF_SERVER"
}

ldap_cmd() {
	OUTFILE="$1"
	BASEDN="$2"
	SEARCH="$3"
	COMMENT="$4"
#	debug OUTFILE BASEDN SEARCH
	echo "#------------------------------------------------#" >> $OUTFILE
	echo "# $LDAP_BIN ${AUTH_OPTIONS} -b \"$BASEDN\" -s sub \"$SEARCH\"" | sed -e "s/-w $NCSVR_PASS/-W/" >> $OUTFILE
	if [ -n "$COMMENT" ]; then
		log_summary "$COMMENT" "$SEARCH"
	else
		log_summary "Query" "$SEARCH"
	fi
	$LDAP_BIN ${AUTH_OPTIONS} -b "$BASEDN" -s sub "$SEARCH" >> $OUTFILE 2>&1
	return $?
}

ldap_check() {
	$LDAP_BIN ${AUTH_OPTIONS} -b "" -s base "objectclass=*" >/dev/null 2>&1
	if [ $? -eq 0 ]; then
		log_summary "LDAPS Connection" "Success"
		return 0
	else
		log_summary "LDAPS Connection" "FAILED"
		log_summary "Preferred Server" "$PREF_SERVER"
		log_summary "DN" "$NCSVR_ADMIN"
		$LDAP_BIN ${AUTH_OPTIONS} -b "" -s base "objectclass=*"
		log_detail
		exit 1
	fi
}

show_summary() {
	FMT_CURRENT="%-30s %s\n"
	log_detail
	log_detail "==[ Summary ]======================================="
	log_detail
	log_summary "Volume Resources Analyzed:" "$TOTAL_VOLRES"
	log_summary "Volume Resources with Errors:" "$TOTAL_VOLRES_ERRORS"
	log_detail
	log_summary "Missing Objects:" "$TOTAL_OBJ"
	log_summary "Missing Attributes:" "$TOTAL_ATTR"
	log_summary "Mismatched Object Links:" "$TOTAL_LINK"
	log_summary "Resource Status Errors:" "$TOTAL_STAT"
	log_summary "TOTAL ERRORS:" "$TOTAL_ERRORS"
	log_detail
	log_detail "===================================================="
	log_detail
}

check_attributes() {
	OBJ_TYPE=$1
	OBJ_NAME=$2
	SRC=$3

	case $OBJ_TYPE in
	VolumeResource)
		OBJ_LABEL="Volume Resource"
		ATTRIB_LIST="nCSNCPServer nCSRevision nCSCRMFailoverMode nCSCRMFailbackMode nCSCRMPreferredNodes nCSCRMUnloadTimeout nCSCRMUnloadScript nCSCRMLoadTimeout nCSCRMLoadScript nCSCRMIgnoreQuorum"
	;;
	Server)
		OBJ_LABEL="NCP Server"
		ATTRIB_LIST="nCSNetWareCluster Resource networkAddress"
		if rpm -q novell-cifs &>/dev/null; then
			ATTRIB_LIST="$ATTRIB_LIST nfapCIFSAttach nfapCIFSServerName nfapCIFSServerName"
		fi
	;;
	Pool) 
		OBJ_LABEL="Pool"
		ATTRIB_LIST="nssfsShared nssfsPoolID hostServer hostResourceName"
	;;
	Volume) 
		OBJ_LABEL="Volume"
		ATTRIB_LIST="linuxNCPMountPoint nssfsPool DFS-Volume-Guid hostServer hostResourceName"
	;;
	Cluster) 
		OBJ_LABEL="Cluster"
		ATTRIB_LIST="nCSSharedDiskFlag nCSEmailFilter nCSEmailAddresses nCSPortNumber nCSNetworkAddress nCSRevision nCSCRMResourcePriority nCSCRMQuorumTimeout nCSCRMQuorum nCSGIPCMaxRetransmits nCSGIPCSlaveWatchdog nCSGIPCMasterWatchdog nCSGIPCTolerance nCSGIPCHeartbeat networkAddress"
	;;
	*) 
		ATTRIB_LIST=""
	;;
	esac

	MISSING_COUNT=0
	FOUND_COUNT=0
	MISSING_ATTRIB=''
	ATTRIB_TOTAL=$(echo $ATTRIB_LIST | wc -w)
	for ATTRIBUTE in $ATTRIB_LIST
	do
		FOUND=$(grep -i ^${ATTRIBUTE}: $SRC)
		if [ -n "$FOUND" ]; then
			((FOUND_COUNT++))
		else
			MISSING_ATTRIB="$ATTRIBUTE $MISSING_ATTRIB"
			((MISSING_COUNT++))
		fi
	done
	case $OBJ_TYPE in
	VolumeResource|Server)
		if [ $DN_VOLUME_COUNT -gt 0 ]; then
			ATTRIBUTE="nCSVolumes"
			ATTRIB_TOTAL=$((ATTRIB_TOTAL + DN_VOLUME_COUNT))
			FOUND=$(grep -i ^${ATTRIBUTE}: $SRC | wc -l)
			if [ $FOUND -eq $DN_VOLUME_COUNT ]; then
				((FOUND_COUNT+=FOUND))
			else
				MISSING=$((DN_VOLUME_COUNT-FOUND))
				MISSING_ATTRIB="$ATTRIBUTE $MISSING_ATTRIB"
				((MISSING_COUNT+=MISSING))
			fi
		fi
	esac
	if [ $MISSING_COUNT -gt 0 ]; then
		log_summary " ${OBJ_NAME}" "MISSING $MISSING_COUNT of $ATTRIB_TOTAL $OBJ_LABEL Attributes"
		for ATTRIBUTE in $MISSING_ATTRIB
		do
			log_summary " ${OBJ_NAME}" " Attribute: ${ATTRIBUTE}"
		done
	else
		log_summary " ${OBJ_NAME}" "Found $FOUND_COUNT of $ATTRIB_TOTAL $OBJ_LABEL Attributes"
	fi

	ERR_ATT=$((ERR_ATT + MISSING_COUNT))
	return 0
}

log_detail() {
	echo "$@" | tee -a $LOG_SUMMARY
}

log_summary() {
	printf "$FMT_CURRENT" "$1" "$2" | tee -a $LOG_SUMMARY
}

get_cluster() {
	ldap_cmd $FILE_CLUSTERS "" "(objectclass=nCSNetWareCluster)"
	CLUSTER_COUNT=$(grep -i ^cn: $FILE_CLUSTERS | wc -l)
	if [ $CLUSTER_COUNT -lt 1 ]; then
		((TOTAL_ERRORS++))
		log_summary " Results" "No Cluster Services Objects Found"
		log_detail
		show_summary
		rm -rf $LOGROOT
		exit 0
	fi
	CN_CLUSTER=$(cluster info basic 2>/dev/null | grep 'Cluster:' | awk '{print $2}' | cut -d, -f1)
	if [ -z "$CN_CLUSTER" ]; then
		if [ $CLUSTER_COUNT -eq 1 ]; then
			CN_CLUSTER=$(grep -i ^cn: $FILE_CLUSTERS | awk '{print $2}')
		else
			FILE_CONFIG_CLUSTER="/etc/opt/novell/ncs/clstrlib.conf"
			if [ -s $FILE_CONFIG_CLUSTER ]; then
				DN_CLUSTER=$(grep -A2 clusterDn $FILE_CONFIG_CLUSTER | tail -1 | cut -d\' -f2)
				CN_CLUSTER=$(echo $DN_CLUSTER | sed -e 's/cn=//g;s/,.*//g')
			else
				((TOTAL_ERRORS++))
				log_summary "ERROR" "Cannot determine node's cluster membership"
				log_detail
				show_summary
				rm -rf $LOGROOT
				exit 1
			fi
		fi
	fi

	FILE_NODE="/etc/opt/novell/ncs/nodename"
	[ -s $FILE_NODE ] && CN_NODE=$(cat $FILE_NODE) || CN_NODE='Missing'
	
	DN_CLUSTERS=''
	for I in $(grep -i "^dn:" $FILE_CLUSTERS | awk '{print $2}')
	do
		TMPI=$(echo $I | sed -e 's/cn=//g;s/,.*//g')
		DN_CLUSTERS="$TMPI $DN_CLUSTERS"
	done
	log_summary "Clusters" "$DN_CLUSTERS"

	DN_CLUSTER=$(grep -i "^dn:.*$CN_CLUSTER" $FILE_CLUSTERS | awk '{print $2}')
	if [ -n "$DN_CLUSTER" ]; then
		DN_CLUSTER_PARENT=$(echo $DN_CLUSTER | sed -e "s/cn=${CN_CLUSTER},//")
		#debug CN_CLUSTER DN_CLUSTER DN_CLUSTER_PARENT
		log_summary "Node in Cluster" "$CN_CLUSTER"
		log_summary " Node Name" "$CN_NODE"
		if cluster view | grep -i epoch &>/dev/null; then
			log_summary " Node Status" "Joined"
		else
			log_summary " Node Status" "Not Joined"
		fi
	else
		((TOTAL_ERRORS++))
		log_summary "ERROR" "Cannot determine node's cluster membership"
		log_detail
		show_summary
		rm -rf $LOGROOT
		exit 1
	fi
}

debug() {
	echo
	for i in "$@"
	do
		printf "$i="
		eval echo \${$i}
	done
	echo
}

shadow_volume() {
	RC=1
	MOUNT_POINT=$(echo $MOUNT_ENTRY | awk '{print $3}')
	NCP_CONF_FILE="/etc/opt/novell/ncpserv.conf"
	if [ -s $NCP_CONF_FILE ]; then
		SHADOW_VOLUME=$(grep "^SHADOW_VOLUME.*${MOUNT_POINT}$" $NCP_CONF_FILE)
		if [ -n "$SHADOW_VOLUME" ]; then
			RC=0
		fi
	fi
	#debug MOUNT_ENTRY MOUNT_POINT SHADOW_VOLUME CN_VOLUME_NAME NCP_CONF_FILE
	return $RC
}

validate_required_objects() {
	log_detail "Checking Required Objects"
	if [ -s $FILE_VOLUME_RESOURCE ]; then
		#debug DN_VOLUME_RESOURCE FILE_VOLUME_RESOURCE
		if grep -i "^dn: cn=$CN_VOLUME_RESOURCE" $FILE_VOLUME_RESOURCE &>/dev/null; then
			log_summary " $CN_VOLUME_RESOURCE" "Found Volume Resource Object"
		else
			log_summary " $CN_VOLUME_RESOURCE" "MISSING Volume Resource Object"
			((ERR_OBJ++))
		fi
	fi
	if [ -s $FILE_NCPSERVER ]; then
		if grep -i "^dn: $DN_NCPSERVER" $FILE_NCPSERVER &>/dev/null; then
			log_summary " ${CN_NCPSERVER##cn=}" "Found NCP Server Object"
		else
			log_summary " ${CN_NCPSERVER##cn=}" "MISSING NCP Server Object"
			((ERR_OBJ++))
		fi
	else
		log_summary " NCP Server Object" "Missing"
		((ERR_OBJ++))
	fi
	if [ -s $FILE_POOL ]; then
		if grep -i "^dn: $DN_POOL" $FILE_POOL &>/dev/null; then
			log_summary " ${CN_POOL##cn=}" "Found Pool Object"
		else
			log_summary " ${CN_POOL##cn=}" "MISSING Pool Object"
			((ERR_OBJ++))
		fi
	else
		log_summary " Pool Object" "Missing"
		((ERR_OBJ++))
	fi
	if [ -n "$DN_VOLUME_LIST" ]; then
		for DN_VOLUME in $DN_VOLUME_LIST
		do
			CN_VOLUME=$(echo $DN_VOLUME | cut -d, -f1)
			FILE_VOLUME="${FILE_BASE}_${CN_VOLUME##cn=}_volume.txt"
			if grep -i "^dn: $DN_VOLUME" $FILE_VOLUME &>/dev/null; then
				log_summary " ${CN_VOLUME##cn=}" "Found Volume Object"
			else
				log_summary " ${CN_VOLUME##cn=}" "MISSING Volume Object"
				((ERR_OBJ++))
			fi
		done
	else
		log_summary " Volume Object(s)" "Missing"
		((ERR_OBJ++))
	fi
}

validate_object_attributes() {
	log_detail "Checking Object Attributes"
	if [ -s $FILE_VOLUME_RESOURCE ]; then
		check_attributes VolumeResource "${CN_VOLUME_RESOURCE}" $FILE_VOLUME_RESOURCE 
	else
		log_summary " Unidentified Volume Resource Object" "Skipping Attribute Check"
	fi
	if [ -s $FILE_NCPSERVER ]; then
		check_attributes Server "${CN_NCPSERVER##cn=}" $FILE_NCPSERVER
	else
		log_summary " Unidentified NCP Server Object" "Skipping Attribute Check"
	fi
	if [ -s $FILE_POOL ]; then
		check_attributes Pool "${CN_POOL##cn=}" $FILE_POOL
	else
		log_summary " Unidentified Pool Object" "Skipping Attribute Check"
	fi
	if [ -n "$DN_VOLUME_LIST" ]; then
		for DN_VOLUME in $DN_VOLUME_LIST
		do
			CN_VOLUME=$(echo $DN_VOLUME | cut -d, -f1)
			FILE_VOLUME="${FILE_BASE}_${CN_VOLUME##cn=}_volume.txt"
			check_attributes Volume "${CN_VOLUME##cn=}" $FILE_VOLUME
		done
	else
		log_summary " Unidentified Volume Object(s)" "Skipping Attribute Check"
	fi
}

confirm_object_links() {
	log_detail "Confirming Object Links"
	FMT_PREV=$FMT_CURRENT
	FMT_CURRENT=$FMT_SHORT
	if [ -s $FILE_VOLUME_RESOURCE ]; then
		DN_VR=$(grep -i ^dn: $FILE_VOLUME_RESOURCE | awk '{print $2}')
		DN_VR2NS=$(grep -i ^nCSNCPServer: $FILE_VOLUME_RESOURCE | awk '{print $2}')
		DN_VR2VL_LIST=$(grep -i ^nCSVolumes: $FILE_VOLUME_RESOURCE | awk '{print $2}')
	else
		DN_VR='vr?'
		DN_VR2NS='vr2ns?'
		DN_VR2VL='vr2vl?'
	fi
	if [ -s $FILE_NCPSERVER ]; then
		DN_NS=$(grep -i ^dn: $FILE_NCPSERVER | awk '{print $2}')
		DN_NS2VR=$(grep -i ^Resource: $FILE_NCPSERVER | awk '{print $2}')
		DN_NS2VL_LIST=$(grep -i ^nCSVolumes: $FILE_NCPSERVER | awk '{print $2}')
		DN_NS2CL=$(grep -i ^nCSNetWareCluster: $FILE_NCPSERVER | awk '{print $2}')
	else
		DN_NS='ns?'
		DN_NS2VR='ns2vr?'
		DN_NS2VL='ns2vl?'
		DN_NS2CL='ns2cl?'
	fi
	if [ -s $FILE_POOL ]; then
		DN_PL=$(grep -i ^dn: $FILE_POOL | awk '{print $2}')
		DN_PL2NS=$(grep -i ^hostServer: $FILE_POOL | awk '{print $2}')
	else
		DN_PL='pl?'
		DN_PL2NS='pl2ns?'
	fi
	#debug DN_VR DN_VR2NS DN_VR2VL DN_NS DN_NS2VR DN_NS2VL DN_NS2CL DN_VL DN_VL2NS DN_VL2PL DN_PL DN_PL2NS

	RESULT="(Resource -> NCP)    ${CN_VOLUME_RESOURCE} -> ${CN_NCPSERVER##cn=}"
	if [ "$DN_VR2NS" = "$DN_NS" ]; then
		log_summary " Pass" "$RESULT"
	else
		log_summary " FAILED" "$RESULT"
		((ERR_LNK++))
	fi

	RESULT="(Resource <- NCP)    ${CN_VOLUME_RESOURCE} <- ${CN_NCPSERVER##cn=}"
	if [ "$DN_NS2VR" = "$DN_VR" ]; then
		log_summary " Pass" "$RESULT"
	else
		log_summary " FAILED" "$RESULT"
		((ERR_LNK++))
	fi

	RESULT="(NCP -> Cluster)     ${CN_NCPSERVER##cn=} -> ${CN_CLUSTER##cn=}"
	if [ "$DN_NS2CL" = "$DN_CLUSTER" ]; then
		log_summary " Pass" "$RESULT"
	else
		log_summary " FAILED" "$RESULT"
		((ERR_LNK++))
	fi

	RESULT="(Pool -> NCP)        ${CN_POOL##cn=} -> ${CN_NCPSERVER##cn=}"
	if [ "$DN_PL2NS" = "$DN_NS" ]; then
		log_summary " Pass" "$RESULT"
	else
		log_summary " FAILED" "$RESULT"
		((ERR_LNK++))
	fi

	if [ -n "$DN_VOLUME_LIST" ]; then
		for DN_VOLUME in $DN_VOLUME_LIST
		do
			CN_VOLUME=$(echo $DN_VOLUME | cut -d, -f1)
			FILE_VOLUME="${FILE_BASE}_${CN_VOLUME##cn=}_volume.txt"
			if [ -s $FILE_VOLUME ]; then
				DN_VL=$(grep -i ^dn: $FILE_VOLUME | awk '{print $2}')
				DN_VL2NS=$(grep -i ^hostServer: $FILE_VOLUME | awk '{print $2}')
				DN_VL2PL=$(grep -i ^nssfsPool: $FILE_VOLUME | awk '{print $2}')
			else
				DN_VL='vl?'
				DN_VL2NS='vl2ns?'
				DN_VL2PL='vl2pl?'
			fi
			RESULT="(Resource -> Volume) ${CN_VOLUME_RESOURCE} -> ${CN_VOLUME##cn=}"
			if echo $DN_VR2VL_LIST | grep $DN_VL &>/dev/null; then
				log_summary " Pass" "$RESULT"
			else
				log_summary " FAILED" "$RESULT"
				((ERR_LNK++))
			fi
			RESULT="(Volume -> Pool)     ${CN_VOLUME##cn=} -> ${CN_POOL##cn=}"
			if [ "$DN_VL2PL" = "$DN_PL" ]; then
				log_summary " Pass" "$RESULT"
			else
				log_summary " FAILED" "$RESULT"
				((ERR_LNK++))
			fi
			RESULT="(NCP -> Volume)      ${CN_NCPSERVER##cn=} -> ${CN_VOLUME##cn=}"
			if echo $DN_NS2VL_LIST | grep $DN_VL &>/dev/null; then
				log_summary " Pass" "$RESULT"
			else
				log_summary " FAILED" "$RESULT"
				((ERR_LNK++))
			fi
			RESULT="(NCP <- Volume)      ${CN_NCPSERVER##cn=} <- ${CN_VOLUME##cn=}"
			if [ "$DN_VL2NS" = "$DN_NS" ]; then
				log_summary " Pass" "$RESULT"
			else
				log_summary " FAILED" "$RESULT"
				((ERR_LNK++))
			fi
		done
	else
		RESULT="(Resource -> Volume) ${CN_VOLUME_RESOURCE} -> Missing Volume"
		log_summary " FAILED" "$RESULT"
		RESULT="(Volume -> Pool)     Missing Volume -> ${CN_POOL##cn=}"
		log_summary " FAILED" "$RESULT"
		RESULT="(NCP -> Volume)      ${CN_NCPSERVER##cn=} -> Missing Volume"
		log_summary " FAILED" "$RESULT"
		RESULT="(NCP <- Volume)      ${CN_NCPSERVER##cn=} <- Missing Volume"
		log_summary " FAILED" "$RESULT"
		((ERR_LNK+=4))
	fi
	FMT_CURRENT=$FMT_PREV
}

object_status() {
	FMT_PREV=$FMT_CURRENT
	FMT_CURRENT=$FMT_LONG
	log_detail "Resource Status From Node: $CN_NODE"
	VRENTRY=$(cluster resources | grep $CN_VOLUME_RESOURCE)
	if [ -n "$VRENTRY" ]; then
		VRSTATUS=$(echo $VRENTRY | awk '{print $2}')
		if cluster resources 2>/dev/null | grep -i "Name.*BCC" &>/dev/null; then
			VRNODE=$(echo $VRENTRY | awk '{print $4}')
		else
			VRNODE=$(echo $VRENTRY | awk '{print $3}')
		fi
		case $VRSTATUS in
		*unning) log_summary " Volume Resource: $CN_VOLUME_RESOURCE" "$VRSTATUS on $VRNODE" ;;
		*) log_summary " Volume Resource: $CN_VOLUME_RESOURCE" "$VRSTATUS" ;;
		esac
		TMPVRS=$(echo $VRSTATUS | tr [:upper:] [:lower:])
		[ "$TMPVRS" = "comatose" ] && ((ERR_STAT++))
		[ "$TMPVRS" = "alert" ] && ((ERR_STAT++))
	else
		log_summary " Volume Resource: $CN_VOLUME_RESOURCE" "Missing"
	fi
	NCP_LOAD="/var/opt/novell/ncs/${CN_VOLUME_RESOURCE}.load"
	if [ -s $NCP_LOAD ]; then
		NCP_IP=$(grep -v '^#' $NCP_LOAD | grep add_secondary_ipaddress | awk '{print $3}')
		if ping -c1 -w1 $NCP_IP &>/dev/null; then
			log_summary " NCP Server: ${CN_NCPSERVER##cn=}" "Pinged $NCP_IP"
		else
			log_summary " NCP Server: ${CN_NCPSERVER##cn=}" "Cannot Ping $NCP_IP"
		fi
	else
		log_summary " NCP Server: ${CN_NCPSERVER##cn=}" "Missing Load Script"		
	fi
	CN_POOL_NAME=$(echo $CN_POOL | sed -e "s/cn=${CN_CLUSTER}_//g;s/_POOL$//g")
	CL_POOL_ENTRY=$(cluster pools 2>/dev/null | grep -i "Pool $CN_POOL_NAME")
	if [ -n "$CL_POOL_ENTRY" ]; then
		if echo $CL_POOL_ENTRY | grep -i "Pool ${CN_POOL_NAME}.*Not Active" &>/dev/null; then
			log_summary " Pool: ${CN_POOL##cn=}" "Not Active"
		else
			ACTIVE_NODE=$(echo $CL_POOL_ENTRY | awk '{print $NF}')
			log_summary " Pool: ${CN_POOL##cn=}" "Active on $ACTIVE_NODE"
		fi
	else
		if mount 2>/dev/null | grep -i "nsspool.*name=$CN_POOL_NAME" &>/dev/null; then
			log_summary " Pool: ${CN_POOL##cn=}" "Mounted, Not Active"
		else
			log_summary " Pool: ${CN_POOL##cn=}" "Not Mounted"
		fi
	fi
	if [ -n "$DN_VOLUME_LIST" ]; then
		for DN_VOLUME in $DN_VOLUME_LIST
		do
			CN_VOLUME=$(echo $DN_VOLUME | cut -d, -f1)
			CN_VOLUME_NAME=$(echo $CN_VOLUME | sed -e "s/cn=${CN_CLUSTER}_//g")
			MOUNT_ENTRY=$(mount 2>/dev/null | grep -i "nssvol.*name=$CN_VOLUME_NAME[\,)]")
			if [ -n "$MOUNT_ENTRY" ]; then
				if shadow_volume; then
					log_summary " Volume: ${CN_VOLUME##cn=}" "Mounted (Shadow Volume)"
				else
					log_summary " Volume: ${CN_VOLUME##cn=}" "Mounted"
				fi
			else
				log_summary " Volume: ${CN_VOLUME##cn=}" "Not Mounted${SHADOW}"
			fi
		done
	fi
}


##############################################################################
# main
##############################################################################

mkdir -p $LOGROOT
chmod 700 $LOGROOT
title
get_credentials
ldap_check
get_cluster
ERR_ATT=0
check_attributes Cluster "${CN_CLUSTER}" $FILE_CLUSTERS
TOTAL_ERRORS=$((TOTAL_ERRORS + ERR_ATT))
TOTAL_ATTR=$((TOTAL_ATTR + ERR_ATT))

SEQ_VR=0
FILE_VOLUME_RESOURCES="${LOGROOT}/${CN_CLUSTER}_vr$((SEQ_VR++))_volume-resources.txt"
ldap_cmd "$FILE_VOLUME_RESOURCES" "$DN_CLUSTER" "(objectclass=nCSVolumeResource)"

DN_VOLUME_RESOURCE_LIST=$(grep -i ^dn: $FILE_VOLUME_RESOURCES | awk '{print $2}')
#debug DN_VOLUME_RESOURCE_LIST
if [ -z "$DN_VOLUME_RESOURCE_LIST" ]; then
	log_summary " Results" "No Volume Resource Objects Found"
	log_detail
	show_summary
	exit 1
fi
TOTAL_VOLRES=$(echo "$DN_VOLUME_RESOURCE_LIST" | wc -l)
log_summary "Volume Resources:" $TOTAL_VOLRES

TOTAL_VOLRES=0
for DN_VOLUME_RESOURCE in ${DN_VOLUME_RESOURCE_LIST}
do
	FMT_CURRENT=$FMT_QUERY
	CN_VOLUME_RESOURCE=$(echo $DN_VOLUME_RESOURCE | cut -d, -f1)
	CN_VOLUME_RESOURCE="${CN_VOLUME_RESOURCE##cn=}"
	log_detail
	log_detail "----------------------------------------------------"
	log_detail
	log_detail "${CN_VOLUME_RESOURCE} Analyzing Volume Resource"
	ERR_VR=0
	ERR_OBJ=0
	ERR_ATT=0
	ERR_LNK=0
	ERR_STAT=0
	FILE_BASE="${LOGROOT}/${CN_CLUSTER}_vr$((SEQ_VR++))"
	FILE_VOLUME_RESOURCE="${FILE_BASE}_${CN_VOLUME_RESOURCE}_1volres.txt"
	FILE_NCPSERVER="${FILE_BASE}_${CN_VOLUME_RESOURCE}_ncpserver.txt"
	FILE_VOLUMES="${FILE_BASE}_${CN_VOLUME_RESOURCE}_volumes.txt"
	FILE_POOL="${FILE_BASE}_${CN_VOLUME_RESOURCE}_pool.txt"
	CN_NCPSERVER="Missing NCP Server"
	CN_VOLUME="Missing Volume"
	CN_POOL="Missing Pool"

	# Query Volume Resource Objects
	ldap_cmd "$FILE_VOLUME_RESOURCE" "$DN_CLUSTER" "(&(cn=$CN_VOLUME_RESOURCE)(objectClass=nCSVolumeResource))" " Query Volume Resource"

	DN_NCPSERVER=$(grep -i ^nCSNCPServer: $FILE_VOLUME_RESOURCE | awk '{print $2}')
	if [ -n "$DN_NCPSERVER" ]; then
		CN_NCPSERVER=$(echo $DN_NCPSERVER | cut -d, -f1)
		ldap_cmd "$FILE_NCPSERVER" "$DN_CLUSTER_PARENT" "(&($CN_NCPSERVER)(objectClass=ncpServer))" " Query NCP Server"
		ldap_cmd "$FILE_POOL" "$DN_CLUSTER_PARENT" "(&(hostServer=$DN_NCPSERVER)(objectClass=nssfsPool))" " Query Pool"
		ldap_cmd "$FILE_VOLUMES" "$DN_CLUSTER_PARENT" "(&(hostServer=$DN_NCPSERVER)(objectClass=Volume))" " Query Volumes"
		DN_VOLUME_LIST=$(grep -i ^dn: $FILE_VOLUMES | awk '{print $2}')
		DN_VOLUME_COUNT=$(echo "$DN_VOLUME_LIST" | wc -l)
		if [ -n "$DN_VOLUME_LIST" ]; then
			SEQ_VL=1
			for DN_VOLUME in $DN_VOLUME_LIST
			do
				CN_VOLUME=$(echo $DN_VOLUME | cut -d, -f1)
				FILE_VOLUME_ENTRY="${FILE_BASE}_${CN_VOLUME##cn=}_volume.txt"
				ldap_cmd "$FILE_VOLUME_ENTRY" "$DN_CLUSTER_PARENT" "(&(cn=${CN_VOLUME##cn=})(objectClass=Volume))" " Query Volume $((SEQ_VL++))/${DN_VOLUME_COUNT}"				
			done
		fi
		DN_POOL=$(grep -i ^dn: $FILE_POOL | awk '{print $2}')
		[ -z "$DN_POOL" ] && rm $FILE_POOL || CN_POOL=$(echo $DN_POOL | cut -d, -f1)
	else
		DN_VOLUME_COUNT=0
	fi

	FMT_CURRENT=$FMT_OBJ
	validate_required_objects
	validate_object_attributes
	confirm_object_links
	object_status

	TOTAL_OBJ=$((TOTAL_OBJ + ERR_OBJ))
	TOTAL_ATTR=$((TOTAL_ATTR + ERR_ATT))
	TOTAL_LINK=$((TOTAL_LINK + ERR_LNK))
	TOTAL_STAT=$((TOTAL_STAT + ERR_STAT))
	ERR_VR=$((ERR_OBJ + ERR_ATT + ERR_LNK + ERR_STAT))
	TOTAL_ERRORS=$((TOTAL_ERRORS + ERR_OBJ + ERR_ATT + ERR_LNK + ERR_STAT))
	[ $ERR_OBJ -gt 0 ] && log_summary " Missing Objects:" $ERR_OBJ
	[ $ERR_ATT -gt 0 ] && log_summary " Missing Attributes:" $ERR_ATT
	[ $ERR_LNK -gt 0 ] && log_summary " Mismatched Object Links:" $ERR_LNK
	[ $ERR_STAT -gt 0 ] && log_summary " Resource Status Errors:" $ERR_STAT
	if (( ERR_VR )); then
		log_detail "$CN_VOLUME_RESOURCE Volume Resource Status: Errors Found"
	else
		log_detail "$CN_VOLUME_RESOURCE Volume Resource Status: Passed"
	fi
	((TOTAL_VOLRES++))
	[ $ERR_VR -gt 0 ] && ((TOTAL_VOLRES_ERRORS++))
done

show_summary

exit 0

