#!/bin/sh
#
# Copyright (C) 1994-2020 Altair Engineering, Inc.
# For more information, contact Altair at www.altair.com.
#
# This file is part of both the OpenPBS software ("OpenPBS")
# and the PBS Professional ("PBS Pro") software.
#
# Open Source License Information:
#
# OpenPBS is free software. You can redistribute it and/or modify it under
# the terms of the GNU Affero General Public License as published by the
# Free Software Foundation, either version 3 of the License, or (at your
# option) any later version.
#
# OpenPBS is distributed in the hope that it will be useful, but WITHOUT
# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU Affero General Public
# License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
# Commercial License Information:
#
# PBS Pro is commercially licensed software that shares a common core with
# the OpenPBS software.  For a copy of the commercial license terms and
# conditions, go to: (http://www.pbspro.com/agreement.html) or contact the
# Altair Legal Department.
#
# Altair's dual-license business model allows companies, individuals, and
# organizations to create proprietary derivative works of OpenPBS and
# distribute them - whether embedded or bundled with other software -
# under a commercial license agreement.
#
# Use of Altair's trademarks, including but not limited to "PBS™",
# "OpenPBS®", "PBS Professional®", and "PBS Pro™" and Altair's logos is
# subject to Altair's trademark licensing policies.


if [ $# -eq 1 -a "$1" = "--version" ]; then
   echo pbs_version = 20.0.1
   exit 0
fi

PBS_VERSION=20.0.1
PBS_AES_SWITCH_VER='14.0'

get_db_user() {
	[ -f "${dbuser_fl}" ] && dbuser_name=`cat "${dbuser_fl}" | tr -d '[:space:]'`
	[ -z "${dbuser_name}" ] && dbuser_name="${PBS_DATA_SERVICE_USER:-postgres}"
	if [ ! -f "${dbuser_fl}" ]; then
		printf "%s" "$dbuser_name" >"${dbuser_fl}"
		chmod 0600 "${dbuser_fl}"
	fi
	cat "${dbuser_fl}"
	return $?
}

chk_dataservice_user() {

	chk_usr="$1"

	id=`id ${chk_usr} 2>&1`
	if [ $? -ne 0 ]; then
		echo "PBS Dataservice user ${chk_usr} does not exist"
		return 1;
	fi

	id=`echo ${id} | cut -c5- | cut -d "(" -f1`
	if [ "$id" = "0" ]; then
		echo "PBS Data Service user should not have root priviledges"
		return 1;
	fi

	# login as ${chk_usr} and try to cd to user home dir
	su - ${chk_usr} -c "cd" > /dev/null 2>&1
	if [ $? -ne 0 ]; then
		echo "Unable to login as user ${chk_usr}. Is the user enabled/home directory accessible?"
		return 1
	fi
	return 0
}

is_cray_xt() {
	if [ -f /proc/cray_xt/cname ] ; then
		return 0
	fi
	return 1
}

set_db_trust_login() {
	datastore_dir=$1
	err=`cp -p ${datastore_dir}/pg_hba.conf ${datastore_dir}/pg_hba.conf.orig 2>&1`
	if [ $? -ne 0 ]; then
		echo "${err}"
		return 1
	fi
	err=`chown ${PBS_DATA_SERVICE_USER} ${datastore_dir}/pg_hba.conf.orig`
	if [ $? -ne 0 ]; then
		echo "${err}"
		return 1
	fi
	err=`sed 's/md5/trust/g' ${datastore_dir}/pg_hba.conf > ${datastore_dir}/pg_hba.conf.new 2>&1`
	if [ $? -ne 0 ]; then
		echo "${err}"
		return 1
	fi
	err=`chown ${PBS_DATA_SERVICE_USER} ${datastore_dir}/pg_hba.conf.new`
	if [ $? -ne 0 ]; then
		echo "${err}"
		return 1
	fi
	err=`mv ${datastore_dir}/pg_hba.conf.new ${datastore_dir}/pg_hba.conf 2>&1`
	if [ $? -ne 0 ]; then
		echo "${err}"
		return 1
	fi
}

revoke_db_trust_login() {
	datastore_dir=$1
	err=`cp -p ${datastore_dir}/pg_hba.conf.orig ${datastore_dir}/pg_hba.conf 2>&1`
	if [ $? -eq 0 ]; then
		rm -f ${datastore_dir}/pg_hba.conf.orig
	else
		echo "${err}"
		return 1
	fi
}

utf8_to_sql_ascii() {
	#
	# If the existing database has UTF-8 character encoding,
	# change the encoding to SQL_ASCII for database upgrade.
	#

	${PBS_EXEC}/sbin/pbs_dataservice status > /dev/null
	if [ $? -eq 0 ]; then
		# stop the dataservice
		${PBS_EXEC}/sbin/pbs_dataservice stop > /dev/null
		if [ $? -ne 0 ]; then
			echo "Error stopping PBS Data Service"
			kill -s SIGTERM `ps -ef | grep "${inst_dir}/bin/postgres" | grep -v grep | awk '{if ($3 == 1) print $2}'`
			return 1
		fi
	fi

	set_db_trust_login "${PBS_HOME}/datastore"
	ret=$?
	if [ $ret -ne 0 ]; then
		return $ret
	fi

	# restart with new credentials
	# redirect the output,
	# Use old pgsql binaries since we
	# are working with old database.
	#

	if [ -d "${PBS_HOME}/pgsql.old" ]; then
		PGSQL_INST_DIR="${PBS_HOME}/pgsql.old"
		export PGSQL_INST_DIR
	elif [ -d "${PBS_HOME}/pgsql.forupgrade" ]; then
		PGSQL_INST_DIR="${PBS_HOME}/pgsql.forupgrade"
		export PGSQL_INST_DIR
	fi

	${PBS_EXEC}/sbin/pbs_dataservice start > /dev/null
	if [ $? -ne 0 ]; then
		echo "Error starting PBS Data Service"
		revoke_db_trust_login "${PBS_HOME}/datastore"
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi
		return 1
	fi

	enc_type=`${PGSQL_INST_DIR}/bin/psql -A -t -p ${PBS_DATA_SERVICE_PORT} -d pbs_datastore -U ${PBS_DATA_SERVICE_USER} -c "select pg_encoding_to_char(encoding) from pg_database where datname = 'pbs_datastore'"`
	if [ ! -z "${enc_type}" -a "${enc_type}" = "UTF8" ]; then
		db_list=`${PGSQL_INST_DIR}/bin/psql -A -t -p ${PBS_DATA_SERVICE_PORT} -d pbs_datastore -U ${PBS_DATA_SERVICE_USER} -c "select datname from pg_database"`
		if [ $? -eq 0 ]; then
			for db_entry in ${db_list}
			do
				update=`${PGSQL_INST_DIR}/bin/psql -A -t -p ${PBS_DATA_SERVICE_PORT} -d pbs_datastore -U ${PBS_DATA_SERVICE_USER} -c "update pg_database set encoding = pg_char_to_encoding('SQL_ASCII') where datname = '$db_entry'"`
				ret=$?
				if [ $ret -ne 0 ]; then
					return $ret
				fi
			done
		fi
	fi

	locale_collate=`${PGSQL_INST_DIR}/bin/psql -A -t -p ${PBS_DATA_SERVICE_PORT} -d pbs_datastore -U ${PBS_DATA_SERVICE_USER} -c "SHOW LC_COLLATE"`
	if [ ! -z ${locale_collate} -a ${locale_collate} != "C" ]; then
		change_locale=1
	fi

	locale_ctype=`${PGSQL_INST_DIR}/bin/psql -A -t -p ${PBS_DATA_SERVICE_PORT} -d pbs_datastore -U ${PBS_DATA_SERVICE_USER} -c "SHOW LC_CTYPE"`
	if [ ! -z ${locale_ctype} -a ${locale_ctype} != "C" ]; then
		change_locale=1
	fi

	# stop the dataservice
	${PBS_EXEC}/sbin/pbs_dataservice stop > /dev/null
	if [ $? -ne 0 ]; then
		echo "Error stopping PBS Data Service"
		kill -s SIGTERM `ps -ef | grep "${PGSQL_INST_DIR}/bin/postgres" | grep -v grep | awk '{if ($3 == 1) print $2}'`
		return 1
	fi

	unset PGSQL_INST_DIR

	revoke_db_trust_login "${PBS_HOME}/datastore"
	ret=$?
	if [ $ret -ne 0 ]; then
		return $ret
	fi

	return 0
}

upgrade_pbs_database() {

	user="${PBS_DATA_SERVICE_USER}"
	inst_dir="${PGSQL_DIR}"
	data_dir="${PBS_HOME}/datastore"

	# Check for existence of old data service directory.
	if [ -d "${PBS_HOME}/pgsql.old" ]; then
		old_inst_dir="${PBS_HOME}/pgsql.old"
		old_data_dir="${PBS_HOME}/datastore.old"
	elif [ -d "${PBS_HOME}/pgsql.forupgrade" ]; then
		old_inst_dir="${PBS_HOME}/pgsql.forupgrade"
		old_data_dir="${PBS_HOME}/datastore.forupgrade"
	fi

	if [ ! -f "${data_dir}/PG_VERSION" ]; then
		echo "Database version file: ${data_dir}/PG_VERSION not found, cannot continue"
		return 1
	fi

	sys_pgsql_ver=$(echo `${PGSQL_BIN}/postgres -V` | awk -F'[ .]' '{ print $3"."$4 }')
	old_pgsql_ver=`cat ${data_dir}/PG_VERSION`
	# strip the minor version from sys_pgsql_ver if old_pgsql_ver does not have minor version (for comparison).
	[[ ! $old_pgsql_ver =~ "." ]] && sys_pgsql_ver=$(echo $sys_pgsql_ver | cut -d '.' -f 1)

	[ ${sys_pgsql_ver%.*} -eq ${old_pgsql_ver%.*} ] && [ ${sys_pgsql_ver#*.} \> ${old_pgsql_ver#*.} ] || [ ${sys_pgsql_ver%.*} -gt ${old_pgsql_ver%.*} ];
	result=$?
	if [ ${result} -eq 0 ]; then
		if [ -d "$PBS_EXEC/pgsql" ]; then
			#Start upgrade process of datastore
			upgrade_db
			return $?
		else
			return 2
		fi
	elif [ "${old_pgsql_ver}" = "${sys_pgsql_ver}" ]; then
		return 0
	fi

	echo "Upgrade from version ${old_pgsql_ver} unsupported"
	return 1
}

upgrade_db() {
	#
	# This routine will insatll a 9.3 database cluster,
	# will perform the pre-upgrade checks for datastore
	# with appropriate authentication management.
	#

	change_locale=0
	server_ctl="${PBS_EXEC}/sbin/pbs_dataservice"
	if [ ! -x "${server_ctl}" ]; then
		echo "${server_ctl} not found"
		return 1
	fi

	if [ ! -x "${PBS_EXEC}/sbin/pbs_ds_password" ]; then
		echo "${PBS_EXEC}/sbin/pbs_ds_password not found"
		return 1
	fi

	if [ ! -x "${inst_dir}/bin/pg_upgrade" ]; then
		echo "${inst_dir}/bin/pg_upgrade not found"
		return 1
	fi

	if [ -z "${old_inst_dir}" ]; then
		echo "Data service directory from previous PBS installation not found,"
		echo "Datastore upgrade cannot continue"
		return 1
	fi

	# Check and convert UTF-8 character encoded database to SQL_ASCII
	utf8_to_sql_ascii
	if [ $? -ne 0 ]; then
		return 1
	fi

	# Backup datastore directory, if backup directory already
	# present then exit.
	if [ -d "${old_data_dir}" ]; then
		echo "Files from previous datastore upgrade found,"
		echo "Datastore upgrade cannot continue"
		return 1
	else
		err=`mv ${data_dir} ${old_data_dir} 2>&1`
		if [ $? -ne 0 ]; then
			echo "${err}"
			return 1
		fi
	fi

	# Invoke the dataservice creation script for pbs
	if [ ! -x "${PBS_EXEC}/libexec/install_db" ]; then
		echo "${PBS_EXEC}/libexec/install_db not found"
		return 1
	fi

	resp=`${PBS_EXEC}/libexec/install_db "upgrade" ${change_locale} 2>&1`
	ret=$?

	if [ ${ret} -ne 0 ]; then
		echo "*** Error initializing the PBS dataservice"
		echo "Error details:"
		echo "$resp"
		return ${ret}
	fi

	# Copy the pg_hba.conf from old cluster.
	err=`cp -p ${old_data_dir}/pg_hba.conf ${data_dir}/pg_hba.conf`
	if [ $? -ne 0 ]; then
		echo "${err}"
		return 1
	fi

	set_db_trust_login "${data_dir}"
	ret=$?
	if [ $ret -ne 0 ]; then
		return $ret
	fi
	set_db_trust_login "${old_data_dir}"
	ret=$?
	if [ $ret -ne 0 ]; then
		return $ret
	fi

	CWD=`pwd`
	cd "${data_dir}"
	#Perform pg_upgrade -c to check if we can upgrade the cluster or not
	err=`su ${user} -c "/bin/sh -c '${PGSQL_LIBSTR} ${inst_dir}/bin/pg_upgrade -b ${old_inst_dir}/bin -B ${inst_dir}/bin -d ${old_data_dir} -D ${data_dir} -c'" 2>&1`
	if [ $? -ne 0 ]; then
		echo "Refer pg_upgrade log files at $PBS_HOME/datastore/pg_upgrade_internal.log,"
		echo "$PBS_HOME/datastore/pg_upgrade_server.log and"
		echo "$PBS_HOME/datastore/pg_upgrade_utility.log for more information"
		revoke_db_trust_login "${data_dir}"
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi

		revoke_db_trust_login "${old_data_dir}"
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi

		return 1
	fi

	#Perform pg_upgrade for database upgrade
	err=`su ${user} -c "/bin/sh -c '${PGSQL_LIBSTR} ${inst_dir}/bin/pg_upgrade -b ${old_inst_dir}/bin -B ${inst_dir}/bin -d ${old_data_dir} -D ${data_dir}'" 2>&1`
	if [ $? -ne 0 ]; then
		echo "Refer pg_upgrade log files at $PBS_HOME/datastore/pg_upgrade_internal.log,"
		echo "$PBS_HOME/datastore/pg_upgrade_server.log and"
		echo "$PBS_HOME/datastore/pg_upgrade_utility.log for more information"
		revoke_db_trust_login "${data_dir}"
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi

		revoke_db_trust_login "${old_data_dir}"
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi

		return 1
	fi

	# start the dataservice
	${server_ctl} start > /dev/null
	if [ $? -ne 0 ]; then
		echo "Error starting PBS Data Service"
		return 1
	fi

	# Optimizer statistics are not transferred by pg_upgrade, so do it manually.
	ENVSTR="PGPORT=${PBS_DATA_SERVICE_PORT}; export PGPORT; PGHOST=${PBS_SERVER}; export PGHOST; PGUSER=${user}; export PGUSER; "
	err=`su ${user} -c "/bin/sh -c '${PGSQL_LIBSTR} ${ENVSTR} ${data_dir}/analyze_new_cluster.sh'"`

	# Update locale of pbs database to C
	if [ ${change_locale} -eq 1 ]; then
		${inst_dir}/bin/psql -A -t -p ${PBS_DATA_SERVICE_PORT} -d pbs_datastore -U ${user} -c "update pg_database set datcollate='C', datctype='C'" > /dev/null
		ret=$?
		if [ $ret -ne 0 ]; then
			return $ret
		fi
	fi

	# stop the dataservice
	${server_ctl} stop > /dev/null
	if [ $? -ne 0 ]; then
		echo "Error stopping PBS Data Service"
		kill -s SIGTERM `ps -ef | grep "${inst_dir}/bin/postgres" | grep -v grep | awk '{if ($3 == 1) print $2}'`
		return 1
	fi
	revoke_db_trust_login "${data_dir}"
	ret=$?
	if [ $ret -ne 0 ]; then
		return $ret
	fi

	cd "${CWD}"
	# Delete old cluster
	err=`${data_dir}/delete_old_cluster.sh`
	ret=$?
	if [ $ret -ne 0 ]; then
		return $ret
	fi
}

chkenv() {
	line=`grep -s "^${1}=" $envfile`
	if [ -z "$line" ]; then
		echo "*** setting ${1}=$2"
		echo "${1}=$2" >> $envfile
	else
		echo "*** leave existing $line"
	fi
}

createdir() {
	if [ -n "$1" -a ! -d "$1" ]; then
		if ! mkdir -p "$1"; then
			echo "*** Could not create $1"
			exit 1
		fi
	fi
	if [ -n "$1" -a -n "$2" ]; then
		chmod "$2" "$1"
	fi
}

createpath() {
	while read mode dir ;do
		createdir "${PBS_HOME}/${dir}" $mode
	done
}

# Return the name of the PBS server host
get_server_hostname() {
	shn=""
	if [ -z "${PBS_PRIMARY}" -o -z "${PBS_SECONDARY}" ] ; then
		if [ -z "${PBS_SERVER_HOST_NAME}" ]; then
			shn="${PBS_SERVER}"
		else
			shn="${PBS_SERVER_HOST_NAME}"
		fi
	else
		shn="${PBS_PRIMARY}"
	fi
	echo ${shn} | awk '{print tolower($0)}'
}

# Ensure the supplied hostname is valid
check_hostname() {
	# Check the hosts file
	getent hosts "${1}" >/dev/null 2>&1 && return 0
	# Check DNS
	host "${1}" >/dev/null 2>&1 && return 0
	return 1
}

backupdir() {
	if [ -d "$1" -a -d "$2" ]; then
		backupdir="$(basename $1).pre.${PBS_VERSION}"
		echo "*** Backing up $1 to ${2}/${backupdir}"
		mv "$1" "${2}/${backupdir}"
	fi
}

#
# Start of the pbs_habitat script
#
conf=${PBS_CONF_FILE:-/etc/pbs.conf}
ostype=`uname 2>/dev/null`
umask 022

echo "***"

# Source pbs.conf to get paths: PBS_EXEC and PBS_HOME and PBS_START_*
. $conf

# Ensure certain variables are set
if [ -z "$PBS_EXEC" ]; then
	echo "*** PBS_EXEC is not set"
	exit 1
fi
if [ ! -d "$PBS_EXEC" ]; then
	echo "*** $PBS_EXEC directory does not exist"
	exit 1
fi
if [ -z "$PBS_HOME" ]; then
	echo "*** PBS_HOME is not set"
	exit 1
fi
if [ ! -d "$PBS_HOME" -o ! "$(/bin/ls -A $PBS_HOME)" ]; then
	echo "*** WARNING: PBS_HOME not found in $PBS_HOME"
	if [ -x "$PBS_EXEC/sbin/pbs_server" ]; then
		component="server"
	elif [ -x "$PBS_EXEC/sbin/pbs_mom" ]; then
		component="execution"
	else
		echo "*** No need to execute pbs_habitat in client installation."
		exit 0
	fi
	${PBS_EXEC}/libexec/pbs_postinstall $component $PBS_VERSION $PBS_EXEC $PBS_HOME "" "sameconf"
fi

# Store the old PBS VERSION for later use
if [ -f "$PBS_HOME/pbs_version" ]; then
	old_pbs_version=`cat $PBS_HOME/pbs_version`
fi

# Get the current PBS version from qstat
if [ -x "$PBS_EXEC/bin/qstat" ]; then
	pbs_version=`"${PBS_EXEC}/bin/qstat" --version | sed -e 's/^.* = //'`
	if [ -z "$pbs_version" ]; then
		echo "*** Could not obtain PBS version from qstat"
		exit 1
	fi
	if [ "$pbs_version" != "$PBS_VERSION" ]; then
		echo "*** Version mismatch."
		echo "*** Build version is $PBS_VERSION"
		echo "*** qstat version is $pbs_version"
		exit 1
	fi
else
	echo "*** File not found: $PBS_EXEC/bin/qstat"
	exit 1
fi

# Perform sanity check on server name in pbs.conf
server_hostname=`get_server_hostname`
[ "$server_hostname" = 'change_this_to_pbs_server_hostname' ] && server_hostname=''
if [ -z "${server_hostname}" ] ; then
	echo "***" >&2
	echo "*** The hostname of the PBS server in ${conf} is invalid." >&2
	echo "*** Update the configuration file before starting PBS." >&2
	echo "***" >&2
	exit 1
fi
check_hostname "${server_hostname}"
if [ $? -ne 0 ]; then
	echo "***" >&2
	echo "*** The PBS server could not be found: $server_hostname" >&2
	echo "*** This value must resolve to a valid IP address." >&2
	echo "***" >&2
	exit 1
fi
server=`echo ${server_hostname} | awk -F. '{print $1}'`

# Check the clienthost entries if present
if [ -f "$PBS_HOME/mom_priv/config" ]; then
	cmd='cat "$PBS_HOME/mom_priv/config"'
	cmd="$cmd | sed -e 's/\t/ /g' -e 's/ \\+/ /g' -e 's/^ //' -e 's/ $//'"
	cmd="$cmd | grep '\\\$clienthost' | cut -d' ' -f2"
	for host in `eval $cmd`; do
		if [ -z "$host" -o "$host" = 'CHANGE_THIS_TO_PBS_SERVER_HOSTNAME' ]; then
			echo "***" >&2
			echo "*** Invalid entry in $PBS_HOME/mom_priv/config" >&2
			echo "*** for clienthost: $host" >&2
			echo "***" >&2
			exit 1
		fi
		if ! check_hostname "${host}"; then
			echo "***" >&2
			echo "*** Invalid entry in $PBS_HOME/mom_priv/config" >&2
			echo "*** for clienthost: $host" >&2
			echo "*** This value must resolve to a valid IP address." >&2
			echo "***" >&2
			exit 1
		fi
	done
fi

if [ "${PBS_START_SERVER:-0}" != 0 ] ; then
	# Source the file that sets PGSQL_LIBSTR
	. "$PBS_EXEC"/libexec/pbs_pgsql_env.sh

	PBS_licensing_loc_file=PBS_licensing_loc

	# On some platforms LD_LIBRARY_PATH etc is not passed on after su
	# so we set it again after the su. PGSQL_LIBSTR contains the line to
	# set the variable again
	export PGSQL_LIBSTR

	dbuser_fl="${PBS_HOME}/server_priv/db_user"
	PBS_DATA_SERVICE_USER=`get_db_user`
	if [ $? -ne 0 ]; then
		echo "Could not retrieve PBS Data Service User"
		exit 1
	fi

	# Do not export the PBS_DATA_SERVICE_USER as a env var
	# since that would cause a false warning message
	# that "deprecated" variable PBS_DATA_SERVICE_USER is
	# being ignored.

	chk_dataservice_user ${PBS_DATA_SERVICE_USER}
	if [ $? -ne 0 ]; then
		exit 1
	fi

	# create the database user file if it does not exist
	if [ ! -f "${dbuser_fl}" ]; then
		printf "${PBS_DATA_SERVICE_USER}" >"${dbuser_fl}"
		chmod 0600 "${dbuser_fl}"
	fi

	server_started=0
	PBS_DATA_SERVICE_PORT=${PBS_DATA_SERVICE_PORT:-"15007"}
	export PBS_DATA_SERVICE_PORT

	create_new_svr_data=1

	# invoke the dataservice creation script for pbs
	if [ ! -x "${PBS_EXEC}/libexec/install_db" ]; then
		echo "${PBS_EXEC}/libexec/install_db not found"
		exit 1
	fi
	resp=`${PBS_EXEC}/libexec/install_db 2>&1`
	ret=$?
	if [ $ret -eq 2 ]; then
		create_new_svr_data=0
	elif [ $ret -ne 0 ]; then
		echo "*** Error initializing the PBS dataservice"
		echo "Error details:"
		echo "$resp"
		exit $ret
	fi

	export PBS_HOME
	export PBS_EXEC
	export PBS_SERVER
	export PBS_ENVIRONMENT

	if [ $create_new_svr_data -eq 0 ]; then
		# datastore directory already exists
		# do database upgrade activities
		upgrade_pbs_database
		ret=$?
		if [ $ret -ne 0 ]; then
			if [ $ret -eq 2 ]; then
				echo "It appears that PostgreSQL has been upgraded independently of PBS."
				echo "The PBS database must be manually upgraded. Please refer to the"
				echo "documentation/release notes for details."
			else
				echo "Failed to upgrade PBS Datastore"
			fi
			exit $ret
		else
			if [ -d "${old_inst_dir}" ]; then
				backupdir "$old_inst_dir" "$PBS_HOME"
				if [ $? -ne 0 ]; then
					echo "Failed to backup $old_inst_dir, please follow the below instructions:"
					echo "*** Backup "$old_inst_dir" if you need to downgrade pgsql later on."
					echo "*** For future upgrades to be successful run the below command."
					echo "*** cp -pr ${PBS_EXEC}/pgsql ${PBS_HOME}/pgsql.forupgrade"
				else
					echo "*** ${PBS_HOME}/$(basename ${old_inst_dir}).pre.${PBS_VERSION} may need to be manually removed if you do not wish to downgrade PBS."
				fi
			fi
		fi
		# do schema upgrade activities
		set_db_trust_login "${PBS_HOME}/datastore"
		${PBS_EXEC}/libexec/pbs_schema_upgrade ${PBS_DATA_SERVICE_PORT} ${PBS_DATA_SERVICE_USER}
		ret=$?
		if [ $ret -ne 0 ]; then
			revoke_db_trust_login "${PBS_HOME}/datastore"
			echo "Failed to upgrade PBS Datastore"
			exit $ret
		fi
		# We need to regenerate the db_password file since we have changed encryption/decryption
		# library from DES to AES in PBS Version PBS_AES_SWITCH_VER
		if [ "$old_pbs_version" \< "${PBS_AES_SWITCH_VER}" ] ;then
			rm -f  "${PBS_HOME}/server_priv/db_password"
			err=`${PBS_EXEC}/sbin/pbs_ds_password -r`
			if [ $? -ne 0 ]; then
				echo $err
				echo "Error setting password for PBS Data Service"
				${server_ctl} stop > /dev/null 2>&1
				revoke_db_trust_login "${PBS_HOME}/datastore"
				exit 1
			fi
		fi
		revoke_db_trust_login "${PBS_HOME}/datastore"
	else
		# Upgrade from FS version to DB
		if [ -f ${PBS_HOME}/server_priv/serverdb ] ; then
			# migrate filesystem data into db version
			${PBS_EXEC}/sbin/pbs_server -t updatedb > /dev/null
			ret=$?
			if [ $ret -ne 0 -o -f ${PBS_HOME}/server_priv/serverdb ] ; then
				echo "*** Error during overlay upgrade"
				exit $ret
			fi
			create_new_svr_data=0
		fi
	fi

	if [ $create_new_svr_data -eq 1 ] ; then
		echo "*** Setting default queue and resource limits."
		echo "***"

		${PBS_EXEC}/sbin/pbs_server -t create > /dev/null
		ret=$?
		if [ $ret -ne 0 ]; then
			echo "*** Error starting pbs server"
			exit $ret
		fi
		server_started=1
		if is_cray_xt; then
			${PBS_EXEC}/bin/qmgr <<-EOF > /dev/null
				set server restrict_res_to_release_on_suspend = ncpus
			EOF
		fi
		tries=3
		while [ $tries -ge 0 ]
		do
			${PBS_EXEC}/bin/qmgr <<-EOF > /dev/null
				create queue workq
			EOF
			ret=$?
			if [ $ret -eq 0 ]; then
				break
			fi
			tries=$((tries-1))
			sleep 2
		done
		${PBS_EXEC}/bin/qmgr <<-EOF > /dev/null
			set queue workq queue_type = Execution
			set queue workq enabled = True
			set queue workq started = True
			set server default_queue = workq
		EOF
		if [ -f ${PBS_HOME}/server_priv/$PBS_licensing_loc_file ]; then
			read ans < ${PBS_HOME}/server_priv/$PBS_licensing_loc_file
			echo "*** Setting license file location(s)."
			echo "***"
			${PBS_EXEC}/bin/qmgr <<-EOF > /dev/null
				set server pbs_license_info = $ans
			EOF
			if ! is_cray_xt; then
				rm -f ${PBS_HOME}/server_priv/$PBS_licensing_loc_file	# clean up after INSTALL
			fi
		fi
	else
		# the upgrade case:  serverdb already exists, but license file
		# information is new
		if [ -f ${PBS_HOME}/server_priv/$PBS_licensing_loc_file ]; then
			read ans < ${PBS_HOME}/server_priv/$PBS_licensing_loc_file
			echo "*** Setting license file location(s)."
			echo "***"
			${PBS_EXEC}/sbin/pbs_server > /dev/null
			${PBS_EXEC}/bin/qmgr <<-EOF > /dev/null
				set server pbs_license_info = $ans
			EOF
			if ! is_cray_xt; then
				rm -f ${PBS_HOME}/server_priv/$PBS_licensing_loc_file	# clean up after INSTALL
			else
				${PBS_EXEC}/bin/qmgr <<-EOF > /dev/null
					set server restrict_res_to_release_on_suspend += ncpus
				EOF
			fi
			server_started=1
		fi
	fi

	if [ $PBS_START_MOM != 0 ]; then
		if [ $server_started -eq 0 ]; then
			${PBS_EXEC}/sbin/pbs_server > /dev/null
			server_started=1
		fi

		if ${PBS_EXEC}/bin/pbsnodes localhost > /dev/null 2>&1 ||
		   ${PBS_EXEC}/bin/pbsnodes $server > /dev/null 2>&1; then
			:
		else
			# node $server is not already available, create
			${PBS_EXEC}/bin/qmgr <<-EOF > /dev/null
				create node $server
			EOF
		fi
	fi

	if [ $server_started -eq 1 ]; then
		${PBS_EXEC}/bin/qterm
	fi
fi

#
# For overlay upgrades PBS_START_MOM will be disabled per the install
# instructions. There may still be job and task files present that
# require updating.
#
if [ -d ${PBS_HOME}/mom_priv/jobs ]; then
	upgrade_cmd="${PBS_EXEC}/sbin/pbs_upgrade_job"
	if [ -x ${upgrade_cmd} ]; then
		total=0
		upgraded=0
		for file in ${PBS_HOME}/mom_priv/jobs/*.JB; do
			if [ -f ${file} ]; then
				${upgrade_cmd} -f ${file}
				if [ $? -ne 0 ]; then
					echo "Failed to upgrade ${file}"
				else
					upgraded=`expr ${upgraded} + 1`
				fi
				total=`expr ${total} + 1`
			fi
		done
		if [ ${total} -gt 0 ]; then
			echo "Upgraded ${upgraded} of ${total} job files."
		fi
	else
		echo "WARNING: $upgrade_cmd not found!"
	fi
fi

# Update the version file at the very end, after everything else succeeds.
# This allows it to be re-run, in case a previous update attempt failed.
# Clobber the existing pbs_version file and populate it with current version.
# This will prevent updating PBS_HOME if this same version is re-installed
#
echo "${pbs_version}" >"$PBS_HOME/pbs_version"

# Copy pgsql directory to PBS_HOME (as pgsql.forupgrade) for it's future upgrade
[ ! -d "${PBS_HOME}/pgsql.forupgrade" -a -d "${PBS_EXEC}/pgsql" -a -d "${PBS_HOME}" ] && cp -pr --no-preserve=timestamps "${PBS_EXEC}/pgsql" "${PBS_HOME}/pgsql.forupgrade" 2>&1

echo "*** End of ${0}"
exit 0
