#!/bin/bash
#
# cs_prepare_simplestack
#
# (c) 2016 SUSE Linux GmbH, Germany. Author: L.Pinne.
# GNU General Public License v2. No warranty.
#
# Version: 2016-12-05
#

###
# define variables

EXE="$0"
ERR="/dev/null"
CFG="/etc/ClusterTools2/cs_prepare_simplestack"
test -s $CFG && source $CFG

# TODO get variables from config
# this host
HOST_A=${2:-"se01demo"}
# other host
HOST_B=${3:-"se02demo"}
# sap sid
SID=${4:-"C11"}
#
sid="${SID,,}"
# domain
DOMAIN="tam-lab.suse.de"
# this ipaddr
IPADR_A=${5:-"192.168.201.181"}
# other ipaddr
IPADR_B=${6:-"192.168.201.182"}
# virtual ip
IPADR_DB=${7:-"192.168.201.188"}
IPADR_CI=${8:-"192.168.201.187"}
IPADR_AS=${9:-"192.168.201.186"}
# shared disk, f.e. for sbd and data
DEV_LUN="/dev/sda"
# sbd disk
DEV_SBD="/dev/sda1"
# data disk
DEV_DAT="/dev/sda2"
#
NETWORK=$( echo $IPADR_A | awk -F. '{print $1"."$2"."$3".0" }' )
# ntp server
NTPSRV="ntp.suse.de"
# OS files to snyc
CFILES="
/root/.ssh/id_dsa
/root/.ssh/id_dsa.pub
/root/.ssh/authorized_keys
/etc/hosts
/etc/ntp.conf
/etc/mdadm.conf
/etc/sysctl.d/sap-nw.conf
/etc/default/grub
/etc/security/limits.conf
/etc/systemd/system/sapinit.service.d/type.conf
/etc/modules-load.d/softdog.conf
/etc/sysconfig/sbd
/etc/corosync/corosync.conf
/etc/corosync/authkey
/etc/lvm/lvm.conf
/clusterconf/$SID/mdadm.conf
"
# SAP files to sync
SFILES="
/etc/opt/sdb
/etc/services
/usr/sap/sapservices
/home/${sid}adm/
/home/sqd${sid}/
/home/sdb/
/home/sapadm/
/sapdb/programs/lib/
/sapdb/clients/$SID/runtime/
/var/lib/sdb/
/root/passwd.nwinstall
/root/shadow.nwinstall
/root/group.nwinstall
"
#
SFILES_LN=$(echo $SFILES | tr -d "\n")
#

function hide_old(){
# just to hide the old variables
CIB_BOOTS="
configure \
property cib-bootstrap-options: \
 no-quorum-policy="ignore" \
 stonith-enabled="true" \
 stonith-action="reboot" \
 have-watchdog=false \
 cluster-infrastructure=corosync
"
CIB_DEF_RSC="
configure \
rsc_defaults \$id="rsc_default-options" \
 resource-stickiness="1000" \
 migration-threshold="5" \
 failure-timeout="86400"
"
CIB_DEF_OP="
configure \
 op_defaults \$id="op_defaults-options" \
 timeout="120"
"
}
#
CIB_BOOTS="
\\n
configure \
property cib-bootstrap-options: \
 no-quorum-policy="ignore" \
 stonith-enabled="true" \
 stonith-action="reboot" \
 have-watchdog=false \
 cluster-infrastructure=corosync
\\n
configure \
rsc_defaults \$id="rsc_default-options" \
 resource-stickiness="1000" \
 migration-threshold="5" \
 failure-timeout="86400"
\\n
configure \
 op_defaults \$id="op_defaults-options" \
 timeout="120"
\\n
"
#
CIB_RES_SBD="
\\n
configure \
 primitive rsc_stonith_sbd stonith:external/sbd \
 params pcmk_delay_max="15"
\\n
"
#
CIB_RES_SIMPLE="
\\n
configure \
primitive rsc_fs_${SID}_sapdb Filesystem \
 params device="/dev/sapvg/sapdb" directory="/sapdb" fstype=xfs options="noatime" \
 op stop timeout=300s interval=0 \
 op monitor interval=30s timeout=130s \
 meta target-role=Stopped
\\n
configure \
primitive rsc_fs_${SID}_sapmnt Filesystem \
 params device="/dev/sapvg/sapmnt" directory="/sapmnt" fstype=xfs options="noatime" \
 op stop timeout=300s interval=0 \
 op monitor interval=30s timeout=130s \
 meta target-role=Stopped
\\n
configure \
primitive rsc_fs_${SID}_usrsap Filesystem \
 params device="/dev/sapvg/usrsap" directory="/usr/sap" fstype=xfs options="noatime" \
 op stop timeout=300s interval=0 \
 op monitor interval=30s timeout=130s \
 meta target-role=Stopped
\\n
configure \
primitive rsc_lvm_${SID}_sapvg LVM \
 params volgrpname=sapvg \
 op monitor interval=130s timeout=130s \
 op stop timeout=130s interval=0 on_fail=fence \
 meta target-role=Stopped
\\n
configure \
primitive rsc_md_${SID}_md0 Raid1 \
 params raidconf="/clusterconf/${SID}/mdadm.conf" raiddev="/dev/md0" \
 op stop timeout=130s interval=0 on_fail=fence \
 meta target-role=Stopped
\\n
configure \
primitive rsc_ip_${SID}_db${sid} IPaddr2 \
 params ip=${IPADR_DB} cidr_netmask=24 nic=eth0 \
 op monitor interval=10s timeout=20s on_fail=restart \
 meta target-role=Stopped
\\n
configure \
primitive rsc_ip_${SID}_ci${sid} IPaddr2 \
 params ip=${IPADR_CI} cidr_netmask=24 nic=eth0 \
 op monitor interval=10s timeout=20s on_fail=restart \
 meta target-role=Stopped
\\n
configure \
primitive rsc_ip_${SID}_as${sid} IPaddr2 \
 params ip=${IPADR_AS} cidr_netmask=24 nic=eth0 \
 op monitor interval=10s timeout=20s on_fail=restart \
 meta target-role=Stopped
\\n
configure \
primitive rsc_sap_${SID}_ASCS00_as${sid} SAPInstance \
 params InstanceName="${SID}_ASCS00_as${sid}" AUTOMATIC_RECOVER="true" \
 op monitor interval=120s timeout=60s start_delay=120s \
 op start timeout=180s \
 op stop timeout=240s on_fail="block" \
 meta target-role=Stopped
\\n
configure \
primitive rsc_sap_${SID}_DVEBMGS01_ci${sid} SAPInstance \
 params InstanceName="${SID}_DVEBMGS01_ci${sid}" AUTOMATIC_RECOVER="true" \
 op monitor interval=180s timeout=60s start_delay=240s \
 op start timeout=240s \
 op stop timeout=240s on_fail="block" \
 meta target-role=Stopped
\\n
configure \
primitive rsc_sapdb_${SID} SAPDatabase \
 params SID="${SID}" DBTYPE="ADA" \
 op monitor interval=120s timeout=60s start_delay=180s \
 op start timeout=1800s \
 op stop timeout=1800s \
 meta target-role=Stopped
\\n
configure \
group grp_sap_${SID} \
 rsc_md_${SID}_md0 rsc_lvm_${SID}_sapvg rsc_fs_${SID}_sapdb rsc_fs_${SID}_sapmnt rsc_fs_${SID}_usrsap \
 rsc_ip_${SID}_db${sid} rsc_ip_${SID}_ci${sid} rsc_ip_${SID}_as${sid} \
 rsc_sap_${SID}_ASCS00_as${sid} rsc_sapdb_${SID} rsc_sap_${SID}_DVEBMGS01_ci${sid} \
 meta target-role=Stopped
\\n
configure \
location loc_grp_sap_${SID}_on_${HOST_A} grp_sap_${SID} 1000: ${HOST_A}
\\n
"
#
CFGVAR="
SID
sid
HOST_A
HOST_B
DOMAIN
IPADR_A
IPADR_B
NETWORK
IPADR_DB
IPADR_CI
IPADR_AS
NTPSRV
DEV_LUN
DEV_SBD
DEV_DAT
CFILES
SFILES
CIB_BOOTS
CIB_DEF_RSC
CIB_DEF_OP
CIB_RES_SBD
CIB_RES_SIMPLE
SAP_NETW_RPMS_INPATTERN
SAP_NETW_PATTERN_NEEDED
SAP_NETW_PATTERN_TABOO
SAP_NETW_VERSION
SUSE_NETW_RPMS_ADDITIONAL
SUSE_NETW_PATTERN_NEEDED
"


function show_help(){
 echo "usage: $(basename $0) --help | --version | --listfunc | --writecfg"
 echo "usage: $(basename $0) --1stnode HOST_A HOST_B SID IPADR_A IPADR_B IPADR_DB IPADR_CI IPADR_AS"
 echo "usage: $(basename $0) --2ndnode HOST_A"
 echo "usage: $(basename $0) --nwinstall HOST_A HOST_B SID IPADR_A IPADR_B IPADR_DB IPADR_CI IPADR_AS"
 echo "usage: $(basename $0) --sapsimple HOST_A HOST_B SID IPADR_A IPADR_B IPADR_DB IPADR_CI IPADR_AS"
 echo
 echo " --help		show help."
 echo " --version	show version."
 echo " --listfunc	show functions."
 echo " --writecfg	show variables."
 echo " --1stnode	prepare 1st node."
 echo " --2ndnode	prepare 2nd node. Needs 1st node already prepared."
 echo " --nwinstall	show hints for Netweaver installation on 1st node."
 echo " --sapsimple	show simplified CIB for Netweaver simplestack."
 echo
}


function write_cfg(){
 for c in $CFGVAR ; do
	echo "${c}=\""
	echo ${!c} | tr " " "\n"
	echo "\""
	echo "#"	
 done
}


###
# OS disk housekeeping

function make_cleanup(){
#
 snapper list
 N=$(snapper list | wc -l ); echo $N
 while [ $N -gt 1 ]; do snapper rm --sync $N; N=$[ $N - 1 ] ; done
 echo "btrfs balance start -dusage=100 /"
 btrfs balance start -dusage=100 /

 echo "snapper list"
 snapper list
 echo
}


function make_snapshot(){
#
 snapper create -t single -d "after cs_prepare_simplestack"

 echo "snapper list"
 snapper list
 echo
}


function make_supportconfig(){
#
 tar -czf /root/$HOST.cfiles.tgz $CFILES

 echo "tar -ztvf /root/$HOST.cfiles.tgz"
 tar -ztvf /root/$HOST.cfiles.tgz
 echo
 supportconfig
 mv /var/log/nts_*tbz /root/
}
###


###
# OS software selection

SAP_NETW_RPMS_INPATTERN="
autoyast2-installation
bc
expect
findutils-locate
krb5-32bit
krb5-client
libgcc_s1
libicu52_1
libltdl7
libnuma1
libopenssl0_9_8
libpng12-0
libssh2-1
libstdc++6
libuuid1
nfs-client
ntp
numactl
python-curses
python-pycurl
sapconf
sudo
syslog-ng
sysstat
tcsh
tuned
unixODBC
uuidd
xfsprogs
"
#
SUSE_NETW_RPMS_ADDITIONAL="
ClusterTools2
xfsdump
rear116
librdmacm1
libibverbs1
perl-TermReadLine-Gnu
"
#
SAP_NETW_VERSION="
kernel:3.12.51-60.20
uuidd:2.25-22.1
util-linux:2.25-22.1
"
#
SAP_NETW_PATTERN_NEEDED="
Minimal
base
x11
sap-nw
sap_server
Basis-Devel
"
#
SAP_NETW_PATTERN_TABOO="
apparmor
oracle_server
sap-hana
"
#
SUSE_NETW_PATTERN_NEEDED="
ha_sles
"
#


function check_software(){
 RPMS_INSTALLED=$(rpm -qa)
 PATTERNS_INSTALLED=$(timeout 60 zypper se -t pattern | awk -F"|" '$1="i" {print $2}' )
 echo $PATTERNS_INSTALLED
 # function chk_swpkg with argument call by name
 echo "### SAP_NETW_RPMS_INPATTERN"
 for f in $SAP_NETW_RPMS_INPATTERN ; do
        echo $RPMS_INSTALLED | tr ' ' '\n' | grep ^${f}-[0-9] 2>/dev/null ||\
        echo "Missing: $f"
 done | sort
 echo
 echo "### SAP_NETW_RPMS_ALSONEEDED"
 for f in $SAP_NETW_RPMS_ALSONEEDED ; do
        echo $RPMS_INSTALLED | tr ' ' '\n' | grep ^${f}-[0-9] 2>/dev/null ||\
        echo "Missing: $f"
 done | sort
 echo
 echo "### SUSE_NETW_RPMS_ADDITIONAL"
 for f in $SUSE_NETW_RPMS_ADDITIONAL ; do
        echo $RPMS_INSTALLED | tr ' ' '\n' | grep ^${f}-[0-9] 2>/dev/null ||\
        echo "Missing: $f"
 done | sort
 echo
}


function make_software(){
 timeout 60 zypper ref

 echo "### SAP_NETW_PATTERN_TABOO"
 for f in $SAP_NETW_PATTERN_TABOO ; do
        timeout 60 zypper rm -t pattern $f
 done
 echo
 echo "### SAP_NETW_PATTERN_NEEDED"
 for f in $SAP_NETW_PATTERN_NEEDED ; do
        timeout 60 zypper in -t pattern $f
 done
 echo
 echo "### SUSE_NETW_PATTERN_NEEDED"
 for f in $SUSE_NETW_PATTERN_NEEDED ; do
        timeout 60 zypper in -t pattern $f
 done
 echo

 echo "### SUSE_NETW_RPMS_ADDITIONAL"
 for f in $SUSE_NETW_RPMS_ADDITIONAL ; do
        timeout 60 zypper --non-interactive in --no-recommends --auto-agree-with-licenses $f
 done
 echo

 timeout 60 zypper --non-interactive --no-gpg-checks up -t patch --no-recommends --auto-agree-with-licenses
}
###


###
# OS basic setup

function make_rsync(){
#
 echo "rsync -av root@${HOST_A}:${HOST_A}.cfiles.tgz /root/"
 rsync -e 'ssh -o StrictHostKeyChecking=no' -av root@${HOST_A}:${HOST_A}.cfiles.tgz /root/
 tar -C / -zxf /root/${HOST_A}.cfiles.tgz

 echo "ls -l ..."
 ls -l $CFILES
 echo
}


function make_sshkey(){
#
 cp -a /root/.ssh/id_dsa /root/.ssh/id_dsa.BAK
 cp -a /root/.ssh/id_dsa.pub /root/.ssh/id_dsa.pub.BAK
 cp -a /root/.ssh/authorized_keys /root/.ssh/authorized_keys.BAK
 ssh-keygen -t dsa -f /root/.ssh/id_dsa -N ""
 cat /root/.ssh/id_dsa.pub >>/root/.ssh/authorized_keys
 chmod 600 /root/.ssh/authorized_keys

 echo "ls -l /root/.ssh/"
 ls -l /root/.ssh/
 echo
}


function make_limits(){
#
# TODO do something
 echo "grep ... /etc/security/limits.conf"
 grep -e "^@sapsys" -e "^@sdba" -e "^@dba" /etc/security/limits.conf
 echo
}


function make_hosts(){
#
 cp -a /etc/hosts /etc/hosts.BAK
 grep -v "^127.0.0.2" /etc/hosts.BAK >/etc/hosts
 cat >>/etc/hosts <<EOF
#
$IPADR_A $HOST_A.$DOMAIN $HOST_A
$IPADR_B $HOST_B.$DOMAIN $HOST_B
$IPADR_DB db$sid.$DOMAIN db$sid
$IPADR_CI ci$sid.$DOMAIN ci$sid
$IPADR_PA pa$sid.$DOMAIN pa$sid
#
EOF

 echo "grep -v \"^#\" /etc/hosts"
 grep -v "^#" /etc/hosts | tr -s "\n"
 echo
}


function make_ntp(){
#
 cp -a /etc/ntp.conf /etc/ntp.conf.BAK
 echo server $NTPSRV >>/etc/ntp.conf

 systemctl enable ntpd.service
 systemctl start ntpd.service
 
 echo "grep -v \"^#\" /etc/ntp.conf"
 grep -v "^#" /etc/ntp.conf | tr -s "\n"
 echo
 sleep 3
 echo "ntpq -p"
 ntpq -p
 echo
}


function make_sysctl(){
#
# TODO how interferes this with sapconf, tuned?
 cp -a /etc/sysctl.d/sap-nw.conf /etc/sysctl.d/sap-nw.conf.BAK
 cat >/etc/sysctl.d/sap-nw.conf <<EOF
# /etc/sysctl.d/sap-nw.conf
# for sap netweaver and hana
net.ipv4.tcp_slow_start_after_idle = 0
net.core.somaxconn = 4096
net.ipv4.tcp_max_syn_backlog = 8192
#net.ipv4.ip_local_port_range = "1024 64999"
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_tw_recycle = 1
net.ipv4.tcp_timestamps = 1
net.ipv4.tcp_syn_retries = 8
#net.ipv4.tcp_wmem = 
#net.ipv4.tcp_rmem =
fs.aio-max-nr = 458752
fs.file-max = 20000000
vm.swappiness = 40
vm.pagecache_limit_mb = 0
vm.pagecache_limit_ignore_dirty = 2
vm.max_map_count = 1000000
vm.dirty_bytes = 1073741824
vm.dirty_background_bytes = 134217728
vm.memory_failure_early_kill = 1
vm.overcommit_memory = 1
kernel.shmmax = 9223372036854775807
kernel.sem = 1250 256000 100 8192
kernel.shmall = 1152921504606846720
#
EOF

 systemctl enable systemd-sysctl.service
 systemctl start systemd-sysctl.service

 echo "-p /etc/sysctl.d/sap-nw.conf"
 sysctl -p /etc/sysctl.d/sap-nw.conf
 echo
}


function make_systemd(){
#
 for s in kdump smartd uuidd alsa-restore alsa-state alsa-store SuSEfirewall2 SuSEfirewall2_init nscd auditd slpd pacemaker corosync ; do
	systemctl stop $s.service
	systemctl disable $s.service
 done

 mkdir -p /etc/systemd/system/sapinit.service.d
 cat >/etc/systemd/system/sapinit.service.d/type.conf <<EOF
[Service]
Type=oneshot
EOF

 # TODO syslog-ng
 for s in sysstat tuned sapconf mcelog ntpd systemd-sysctl sapinit hawk sbd ; do
	systemctl enable $s.service
	systemctl start $s.service
 done
 for s in uuidd ; do
	systemctl enable $s.socket
	systemctl start $s.socket
 done
 echo "cat /etc/systemd/system/sapinit.service.d/type.conf"
 cat /etc/systemd/system/sapinit.service.d/type.conf
 echo
}


function make_grub(){
#
 cp -a /etc/default/grub /etc/default/grub.BAK
 sed -i s/GRUB_CMDLINE_LINUX_DEFAULT=\"/GRUB_CMDLINE_LINUX_DEFAULT=\"elevator=noop\ transparent_hugepage=never\ cgroup_disable=memory\ / /etc/default/grub
 grep -v "^#" /etc/default/grub | tr -s "\n"
 grub2-mkconfig -o /boot/grub2/grub.cfg

 echo "grep -v  \"#\" /etc/default/grub"
 grep -v "#" /etc/default/grub
 echo
}


function make_reboot(){
#
 # TODO do checks and ask for input
 sleep 3
 sync
 reboot ; exit
}

###


###
# OS shared storage

make_fdisk(){
#
 cat | fdisk $DEV_LUN <<EOF
n
p
1
2048
+10M
n
p
2
22528
167772159
t
2
fd
w

EOF

 partprobe $DEV_LUN
 echo "fdisk -l $DEV_LUN"
 fdisk -l $DEV_LUN
 echo
}


function make_mdraid(){
#
 cp -a /etc/mdadm.conf /etc/mdadm.conf.BAK
 cat >/etc/mdadm.conf <<EOF
# /etc/mdadm.conf
# mdadm is under control of cluster
DEVICE /dev/null
#
EOF

 echo "cat /etc/mdadm.conf"
 cat /etc/mdadm.conf
 echo

 cp -a /clusterconf/$SID/mdadm.conf /clusterconf/$SID/mdadm.conf.BAK
 mkdir -p /clusterconf/$SID
 cat >/clusterconf/$SID/mdadm.conf <<EOF
# /clusterconf/$SID/mdadm.conf
DEVICE $DEV_DAT
ARRAY /dev/md0 level=raid1 num-devices=1 UUID=@@MD_UUID@@
#
EOF

 mdadm --create /dev/md0 --metadata=1.2 --bitmap=internal --level=1 --raid-devices=1 --force $DEV_DAT
 MD_UUID=$(mdadm --misc --detail --export /dev/md0 |\
	 	awk -F= '$1=="MD_UUID" {print $2}')
 sed -i s/@@MD_UUID@@/$MD_UUID/ /clusterconf/mdadm.conf 

 echo "/proc/mdstat"
 cat /proc/mdstat
 echo
 echo "cat /clusterconf/$SID/mdadm.conf"
 cat /clusterconf/$SID/mdadm.conf
 echo
}


function make_lvm(){
#
 cp -a /etc/lvm/lvm.conf /etc/lvm/lvm.conf.BAK
 awk -F= ' $1=="    filter " {print $1 "= [ \"a|/dev/vda.*|\",\"a|/dev/md.*|\",\"r|/dev/.*|\" ] "}; $1!="    filter " {print}'\
	/etc/lvm/lvm.conf.BAK >/etc/lvm/lvm.conf
 mkinird
 echo "grep -v \"#\" /etc/lvm/lvm.conf"
 grep -v "#" /etc/lvm/lvm.conf | tr -s "\n"
 echo

 pvcreate /dev/md0
 vgcreate -s64M sapvg /dev/md0
 lvcreate -L 10G -n usrsap sapvg
 lvcreate -L 10G -n sapmnt sapvg
 lvcreate -l 100%FREE -n sapdb sapvg

 echo "pvscan"
 pvscan
 echo
 echo "lvscan"
 lvscan | grep sapvg
 echo
}


function make_dirs(){
#
 mkdir -p /usr/sap/$SID /sapmnt /sapdb /var/opt/sapcd /tmp/NW74/{AS,CI,DB}
 ln -s /var/opt/sapcd /sapcd
}


function make_xfs(){
#
 mkfs.xfs /dev/sapvg/usrsap
 mkfs.xfs /dev/sapvg/sapmnt
 mkfs.xfs /dev/sapvg/sapdb

 mount -onoatime /dev/sapvg/usrsap /usr/sap
 mount -onoatime /dev/sapvg/sapmnt /sapmnt
 mount -onoatime /dev/sapvg/sapdb /sapdb

 mkdir /usr/sap/$SID

 echo "mount | grep sapvg"
 mount | grep sapvg
 echo
 echo "(df -h; df -i) | grep sapvg"
 (df -h; df -i) | grep sapvg
 echo
}


function stop_mdraid(){
#
 umount /usr/sap /sapmnt /sapdb
 vgchange -a n /dev/sapvg
 mdadm --manage --stop /dev/md0

 echo "cat /proc/mdstat"
 cat /proc/mdstat
 echo
}
###


###
# SLE-HA setup

function make_softdog(){
#
 cp -a /etc/modules-load.d/softdog.conf /etc/modules-load.d/softdog.conf.BAK
 cat >/etc/modules-load.d/softdog.conf <<EOF
# /etc/modules-load.d/softdog.conf 
softdog
#
EOF

 modprobe softdog

 echo "lsmod | grep softdog"
 lsmod | grep softdog
 echo
}


function make_sbd(){
#
 cp -a /etc/sysconfig/sbd /etc/sysconfig/sbd.BAK
 cat >/etc/sysconfig/sbd <<EOF
# /etc/sysconfig/sbd
SBD_DEVICE="$DEV_SBD"
SBD_WATCHDOG="yes"
SBD_PACEMAKER="yes"
SBD_STARTMODE="clean"
#
EOF

 echo "cat /etc/sysconfig/sbd"
 cat /etc/sysconfig/sbd
 echo

 SBD_DEVICE=$(awk -F= '$1=="SBD_DEVICE" {print $2}' /etc/sysconfig/sbd | tr -d \")
 sbd -d $SBD_DEVICE -1 30 -4 90 create
 echo "sbd -d $SBD_DEVICE dump" 
 sbd -d $SBD_DEVICE dump
 echo
 systemctl enable sbd.service
}


function make_corosync(){
#
 cp -a /etc/corosync/corosync.conf /etc/corosync/corosync.conf.BAK
 cp -a /etc/corosync/authkey /etc/corosync/authkey.BAK 

 cat >/etc/corosync/corosync.conf <<EOF
# /etc/corosync/corosync.conf
totem {
	version: 2
	token: 5000
	fail_recv_const: 5000
	token_retransmits_before_loss_const: 28
	join: 60
	max_messages: 20
	crypto_cipher: none
	crypto_hash: none
	clear_node_high_bit: yes
	transport: udpu
	interface {
		ringnumber: 0
		bindnetaddr: $NETWORK
		mcastport: 5405
		ttl: 1
	}
}
logging {
	fileline: off
	to_syslog: yes
	debug: off
	timestamp: on
	logger_subsys {
		subsys: QUORUM
		debug: off
	}
}
nodelist {
	node {
		ring0_addr: $IPADR_A
		nodeid: 1
	}
	node {
		ring0_addr: $IPADR_B
		nodeid: 2
	}
}
quorum {
	provider: corosync_votequorum
}
#
EOF

 #corosync-keygen

 echo "cat /etc/corosync/corosync.conf"
 cat /etc/corosync/corosync.conf
 echo 

 systemctl start corosync.service
 systemctl start pacemaker.service

 sbd -d $SBD_DEVICE list
 echo
 sleep 3
 echo "timeout 90 crm_mon -1r" 
 timeout 90 crm_mon -1r
 echo
}


function make_cibbasic(){
#
 echo $CIB_BOOTS | crm -f -
 # TODO one single section for all boostrap options
 #echo $CIB_DEF_RES | crm -f - 
 #echo $CIB_DEF_OP | crm -f - 
 echo $CIB_RES_SBD | crm -f -

 echo "crm configure show"
 crm configure show
 echo
}


function make_cibsimple(){
#
 echo -e $CIB_RES_SIMPLE
 #| crm -f -
 
 echo
 echo "crm configure show"
 # crm configure show
 echo

}
###


###
# install Netweaver

function make_nwinstall(){
#
cat <<EOF

 ### Prepare network

 # hostname
 # ip a a $IPADR_DB dev eth0
 # ip a a $IPADR_CI dev eth0
 # ip a a $IPADR_AS dev eth0

 
 ### Prepare filesystem

 # echo "Before activating storage, make sure it is NOT USED on the other host."
 # hostname
 # ssh $HOST_B "hostname ; cat /proc/mdstat"

 # mdadm --assemble --config /clusterconf/$SID/mdadm.conf /dev/md0
 # vgchange -ay sapvg
 # mount -onoatime /dev/sapvg/sapdb /sapdb
 # mount -onoatime /dev/sapvg/sapmnt /sapdb /sapmnt
 # mount -onoatime /dev/sapvg/usrsap /usr/sap

 
 ### Prepare installation source

 # hostname
 # cd /sapcd
 # df -h .
 # du -sh /data/SCT/media/SAP-MEDIA/NW74/{SWPM,sap_kernel,sap_export,MaxDB_7.9_RDBMS___SP8_Build_21_}
 # cp -a /data/SCT/media/SAP-MEDIA/NW74/{SWPM,sap_kernel,sap_export,MaxDB_7.9_RDBMS___SP8_Build_21_} .
 # mkdir -p /tmp/NW74/{AS,CI,DB}; sync


 ### Prepare graphical access

 on hana-02 run: 	  # vncserver :35
 on workstation run:      # vncviever hana-02:35
 on vnc session terminal: # ssh -X se01demo
 

 ### Install ASCS instance

 # cd /tmp/NW74/AS
 # /sapcd/SWPM/sapinst SAPINST_USE_HOSTNAME=asc11
 "SAP Installation Master"
 "-> SAP NetWeaver 7.4 Support Release 2"
 "--> MaxDB"
 "---> SAP Systems"
 "----> Application Server ABAP"
 "-----> Distributed System"
 "------> ASCS Instance"
 "[x] Typical"
 "permission ... [OK]"
 "SAPSID: C11"
 "UC Kernel NW740 SR2: /sapcd/sap_kernel"
 "Password for All User: SuSE1234"
 "Summary ... -> [Next]"
 "Task Progress ...few minutes..."
 "[OK]"
 # sync ; du -sh /sapmnt /usr/sap /sapdb

 ### Install DB instance

 # cd /tmp/NW74/DB
 # /sapcd/SWPM/sapinst SAPINST_USE_HOSTNAME=asc11
 "SAP Installation Master"
 "-> SAP NetWeaver 7.4 Support Release 2"
 "--> MaxDB"
 "---> SAP Systems"
 "----> Application Server ABAP"
 "-----> Distributed System"
 "------> Database Instance"
 "[x] Typical"
 "permission ... [OK]"
 "Profiles Available [x]; Profile directory ..."
 "Password for All Users: SuSE1234"
 "UC Kernel NW740 SR2: /sapcd/sap_kernel"
 "Database ID (DBSID): C11"
 "Installation Export NW740 SR2 (folder EXP1): /sapcd/sap_export"
 "RDBMS MaxDB: /sapcd/MaxDB_7.9__SP8_Build_21_"
 "Database Information ... -> [Next]"
 "Summary ... -> [Next]"
 "Task Progress ...half an hour or so..."
 "[OK]"
 # sync ; du -sh /sapmnt /usr/sap /sapdb


 ### Install PAS instance

 # cd /tmp/NW74/CI
 # /sapcd/SWPM/sapinst SAPINST_USE_HOSTNAME=cic11
 "SAP Installation Master"
 "-> SAP NetWeaver 7.4 Support Release 2"
 "--> MaxDB"
 "---> SAP Systems"
 "----> Application Server ABAP"
 "-----> Distributed System"
 "------> Primary Application Server Instance"
 "[x] Typical"
 "permission ... [OK]"
 "Profile Directory ... [Next]"
 "Password for All Users: SuSE1234"
 "UC Kernel NW740 SR2: /sapcd/sap_kernel"
 "RDBMS MaxDB: /sapcd/MaxDB_7.9__SP8_Build_21_"
 "[x] No SLD destination"
 "Installation Export NW740 SR2 (folder EXP1): /sapcd/sap_export"
 "Secure Storage Individual Key Information - [x] Default Key"
 "DO NOT: [] Install Diagnostics Agent"
 "Summary ... -> [Next]"
 "Task Progress ...some minutes..."
 "[OK]"
 # sync ; du -sh /sapmnt /usr/sap /sapdb


 ### Check SAP Netweaver and MaxDB installation on 1st node

 # hostname
 # ps aux | awk '\$1=="sdb" {print \$1,\$2,\$11}'
 # ps aux | awk '\$1=="${sid}adm" {print \$1,\$2,\$11}'
 # su - ${sid}adm
 # > stopsap r3 DVEBMGS01 ci${sid}
 # > stopdb
 # > stopsap r3 ASCS00 as${sid}
 # > ps aux | awk '\$1=="sdb" {print \$1,\$2,\$11}'
 # > ps aux | awk '\$1=="${sid}adm" {print \$1,\$2,\$11}'
 # > startsap r3 ASCS00 as${sid}
 # > startdb
 # > startsap r3 DVEBMGS01 ci${sid}
 # > ps aux | awk '\$1=="sdb" {print \$1,\$2,\$11}'
 # > ps aux | awk '\$1=="${sid}adm" {print \$1,\$2,\$11}'
 # > stopsap r3 DVEBMGS01 ci${sid}
 # > stopdb
 # > stopsap r3 ASCS00 as${sid}
 # > exit
 # pkill -U sdb
 # pkill -U ${sid}adm
 # pkill -U sapadm
 # rm /home/${sid}adm/.*_$HOST_A.*sh
 # rm /home/sqd${sid}/.*_$HOST_A.*sh
 # TODO collect log and config file


 ### Copy some Netweaver and MaxDB files to local disk, and to 2nd node

 # hostname
 # grep -e"${sid}adm" -e"sapadm" -e"sdb" -e"sqd${sid}" /etc/passwd >/root/passdw.nwinstall
 # grep -e"${sid}adm" -e"sapadm" -e"sdb" -e"sqd${sid}" /etc/shadow >/root/shadow.nwinstall
 # grep -e"${sid}adm" -e"sapadm" -e"sdb" -e"sqd${sid}" /etc/group >/root/group.nwinstall
 # tar -czf /root/$HOST_A.sfiles.tgz $SFILES_LN
 # umount /sapdb /sapmnt /usr/sap
 # vgchange -an sapvg
 # mdadm --stop /dev/md0
 # mount | grep sap
 # tar -C / -zxf /root/$HOST_A.sfiles.tgz
 # scp /root/$HOST_A.sfiles.tgz ${HOST_B}:/root/$HOST_A.sfiles.tgz

 # echo "Be careful with the following. Better do it on $HOST_B locally."
 # ssh $HOST_B "tar -C / -zxf /root/$HOST_A.sfiles.tgz"
 # ssh $HOST_B "tar czf /root/etc.BAK.tgz /etc/"
 # ssh $HOST_B "tar ztf /root/etc.BAK.tgz
 # ssh $HOST_B "cat /root/passwd.nwinstall >>/etc/passwd
 # ssh $HOST_B "cat /root/shadow.nwinstall >>/etc/shadow
 # ssh $HOST_B "cat /root/group.nwinstall >>/etc/group"

 # hostname
 # sync; shutdown -h now; exit


 ### Check SAP Netweaver and MaxDB installation on 2nd node

 # hostname
 # ip a a $IPADR_DB dev eth0
 # ip a a $IPADR_CI dev eth0
 # ip a a $IPADR_AS dev eth0
 
 # echo "Before activating storage, make sure it is NOT USED on the other host."
 # hostname
 # ssh $HOST_B "hostname ; cat /proc/mdstat"
 
 # mdadm --assemble --config /clusterconf/$SID/mdadm.conf /dev/md0
 # cat /proc/mdstat
 # vgchange -ay sapvg
 # mount -onoatime /dev/sapvg/sapdb /sapdb/
 # mount -onoatime /dev/sapvg/sapmnt /sapmnt/
 # mount -onoatime /dev/sapvg/usrsap /usr/sap/
 # df -h | grep sap
 # su - ${sid}adm
 # > startsap r3 ASCS00 as${sid}
 # > startdb
 # > startsap r3 DVEBMGS01 ci${sid}
 # > ps aux | awk '\$1=="sdb" {print \$1,\$2,\$11}'
 # > ps aux | awk '\$1=="${sid}adm" {print \$1,\$2,\$11}'
 # > stopsap r3 DVEBMGS01 ci${sid}
 # > stopdb
 # > stopsap r3 ASCS00 as${sid}
 # > exit
 # pkill -U sdb
 # pkill -U ${sid}adm
 # pkill -U sapadm
 # umount /sapdb /sapmnt /usr/sap
 # vgchange -an sapvg
 # mdadm --stop /dev/md0
 # cat /proc/mdstat

EOF
}


function check_prereq(){
#
# TODO more checks from make_ functions

 uname -a
 cat /proc/cmdline

 rpm -qa | grep -e kernel -e uuidd -e util-linux -e xfs
 # TODO SAP_..NEEED ..
 echo
 ntpq -p | tail -1
 echo
 hostname -f
 ping -c1 $HOST_A | grep "64 bytes"
 ping -c1 $HOST_B | grep "64 bytes"
 ping -c1 as$sid | grep "64 bytes"
 ping -c1 db$sid | grep "64 bytes"
 ping -c1 ci$sid | grep "64 bytes"
 echo
 systemctl cat sapinit.service
 # TODO more systemctl

 # TODO cat /etc/systemd/system/sapinit.service.d/type.conf
 # should be file or directory?
 # TODO varibale for services
 for s in sapinit sapconf tuned uuidd ntpd ; do
	systemctl status $s.service
 done 
 for s in uuidd.socket ; do
	systemctl status $s.service
 done 
 echo
 cs_show_error_patterns | grep -v "=.0"$

}


# TODO # systemctl list-unit-files | grep sap
# sapconf.service                         enabled
# TODO systemctl list-units | grep sap
#sapdb.mount         loaded active mounted   /sapdb
#usr-sap.mount       loaded active mounted   /usr/sap
#sapconf.service     loaded active exited    sapconf
# TODO /etc/systemd/system/sapinit.service.d/

###


###
# main()

case $1 in
 -v|--version)
	echo -n "$(basename $EXE) "
	head -11 $EXE | grep "^# Version: "
	exit
 ;;
 -w|--writecfg)
	write_cfg
	exit
 ;;
 -l|--listfunc)
	grep "^function.*(){" $EXE
	exit
 ;;
 -1|--1stnode)
	make_cleanup
	make_software
	make_cleanup

	make_sshkey
	make_hosts
	make_limits
	make_ntp
	make_sysctl
	make_systemd
	make_grub

	make_dirs
	make_fdisk
	make_mdraid
	make_lvm
	make_xfs
	stop_mdraid

	make_softdog
	make_sbd
	make_corosync
	make_cibbasic
	make_cibsimple

	make_supportconfig
	make_snapshot
	exit
 ;;
 -2|--2ndnode)
	make_cleanup
	make_software
	make_cleanup

	make_rsync

 	make_ntp
	make_sysctl
	make_systemd
	make_grub

	make_dirs

	make_softdog
	make_corosync

	make_supportconfig
	make_snapshot
	exit
 ;;
 -n|--nwinstall)
	make_nwinstall
	exit
 ;;
 -s|--sapsimple|--simplestack)
	make_cibsimple
	exit
 ;;
 *)
	show_help
	exit
 ;;
esac
#
