#!/bin/bash
#
# Warewulf Node Health Check -- Node Onlining Helper
#
# Michael Jennings <mej@lbl.gov>
# 16 September 2011
#
# $Id: node-mark-online 1174 2012-10-30 00:59:27Z mej $
#

# This script is a simple pbsnodes wrapper that the node health check
# can run in the background to mark nodes online after having been
# previously offlined.  It will first obtain the current node state
# information to avoid onlining nodes it didn't originally offline.

PBSNODES="${PBSNODES:-pbsnodes}"
PBSNODES_LIST_ARGS="${PBSNODES_LIST_ARGS:--n -l all}"
PBSNODES_ONLINE_ARGS="${PBSNODES_ONLINE_ARGS:--c -N}"
IGNORE_EMPTY_NOTE="${IGNORE_EMPTY_NOTE:-0}"
LEADER="NHC:"

echo "`date '+%Y%m%d %H:%M:%S'` $0 $*"

HOSTNAME="$1"
NOTE=""

LINE=( $($PBSNODES $PBSNODES_LIST_ARGS $HOSTNAME) )
STATUS="${LINE[1]}"
OLD_NOTE_LEADER="${LINE[2]}"
OLD_NOTE="${LINE[*]:3}"
case $STATUS in
    *offline*)
        # If there is no old note, and we've not been told to ignore that, do not online the node.
        if [[ -z "$OLD_NOTE_LEADER" && "$IGNORE_EMPTY_NOTE" != "1" ]]; then
            echo "$0:  Not onlining $HOSTNAME:  No note set."
            exit 0
        fi
        # If there IS an old note, but it wasn't set by NHC, do not online the node.
        if [[ -n "$OLD_NOTE_LEADER" && "$OLD_NOTE_LEADER" != "$LEADER" ]]; then
            echo "$0:  Not onlining $HOSTNAME:  $OLD_NOTE_LEADER $OLD_NOTE"
            exit 0
        fi
        echo "$0:  Marking $HOSTNAME online and clearing note ($OLD_NOTE_LEADER $OLD_NOTE)"
        exec $PBSNODES $PBSNODES_ONLINE_ARGS '' $HOSTNAME
        ;;
esac
echo "$0:  Skipping $STATUS node $HOSTNAME ($OLD_NOTE_LEADER $OLD_NOTE)"
