#!/bin/bash
#
# cs_show_supportconfig
#
# (c) 2011-2017 SUSE Linux GmbH, Germany.
# (c) 2018-2023 SUSE LLC
# Author: L.Pinne.
# GNU General Public License v2. No warranty.
# http://www.gnu.org/licenses/gpl.html
#
# Version: 2023-10-26 SLES12 0.03
#
# For Supportconfig Script Version: 2.25-237 to 3.0.2-10
# TODO what supportconfig supports sar xz files?
#
# shellcheck disable=SC1090,SC2128,SC2140,SC2125
# SC2140 and SC2125 are needed because of the ongoing pattern declaration
# discussion.
#

EXE=$0
ERR="/dev/null"

CFG="/etc/ClusterTools2/cs_show_supportconfig"
test -s $CFG && source $CFG
test -z "${TEMP}" &&\
	TEMP="/dev/shm/cltl.$RANDOM"

# TODO make DIR= work
DIR=${2:-$PWD}
FUN=${3:-"chk_overview"}
OWD=$PWD

# grep "=\"$" /etc/ClusterTools2/cs_show_supportconfig | tr -d "=\""
# grep "[A-Z]=\"" /etc/ClusterTools2/cs_show_supportconfig | tr -d "=\""
CFGVAR="
ERRCFG
TEMP
HARDW_MOD
NIC_PTRN
SVC_YES
SVC_OFF
NETW_PTRN
TIME_ERR
TIME_PTRN
SOFTW_PTRN
UPDT_PTRN
STOR_PKG
STOR_MOD
SYSC_PTRN
MEM_VMSTAT
MEM_SYSCTL
ORA_SFT_PTRN
ORA_SVC_YES
ORA_SVC_OFF
VRT_SVC_YES
VRT_ERR_PTRN
CLUSTER_PKG
CLUSTER_PRCS
PWVM_PKG
PWVM_MODUL
PWVM_ETC
VMW_PKG
VMW_MODUL
VMW_SVC_YES
VMW_SVC_OFF
VMW_ETC
SAP_SAP_PKG_NOT
HANA_SAP_PKG_YES
HANA_SAP_PKG_NOT
HANA_SUSE_PKG_YES
HANA_VERS_MIN
HANA_SAP_BOOTARG
HANA_SUSE_BOOTARG
HANA_SAP_SYSCTL
HANA_SUSE_SYSCTL
HANA_SAP_SVC_YES
HANA_SAP_SVC_OFF
HANA_SUSE_PTRN
"

function writecfg(){
	echo -e "# $CFG \n# For SLES12.\n#"
	for c in $CFGVAR; do
		echo "${c}=\""
		# shellcheck disable=SC2086
		echo ${!c} | tr " " "\n"
		echo "\""
		echo "#"
	done
}


function echo_msgsep(){
	echo "============================================================ ${1:0:14} ==="
}


function split_file(){
	F=$1
	mkdir -p "${TEMP}"/"$F"
	cp "$F" "${TEMP}"/"${F}"/
	cd "${TEMP}"/"$F" || exit
#	csplit $F /^#==\[ [a-zA-Z] ^C \]==*#$/ {*}
	# shellcheck disable=SC1083
	csplit "$F" /^#==..Co.*===========================#$/ {*} >>$ERR
	cd "${OLDPWD}" || exit
}


function show_sys_scsi(){
	echo "# /usr/bin/systool -vb scsi"
	FIL="sysfs.txt"
	split_file ${FIL}
	f=$(grep -l "^# /usr/bin/systool -vb scsi$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -e "Device.path.=.*/sys/devices/.*target.*/[0-9]" -e "timeout.*=..[1-9]" -e "model.*=" $f
}


function show_sys_fcfabric(){
	echo "# /usr/bin/systool -vc fc_host"
	FIL="sysfs.txt"
	split_file ${FIL}
	f=$(grep -l "^# /usr/bin/systool -vc fc_host$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		( grep -A13 "Class.Device.path.=.\"/sys/devices/" $f
		echo
		#awk -F"=" '$1=="fabric_name" {print $2}' $f | sort -u
		grep port_name $f | sort -f
		echo )
}


function show_pam_sap(){	
# TODO
	echo "pam_limits :"
	f=$(grep -l "^# /etc/security/limits.conf$" "${TEMP}"/"${FIL}"/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		awk '$1=="oracle" || $1~/sapsys/ || $1~/dba/ {print}' $f | sort -u
	echo
# TODO
	echo "/etc/passwd :"
	f=$(grep -l "^# /etc/passwd$" "${TEMP}"/"${FIL}"/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		awk -F: '$5~/SAP.*Administrator/ {print}' $f | sort -u	
	echo
	echo "getent_passwd :"
	f=$(grep -l "^# /usr/bin/getent passwd$" "${TEMP}"/"${FIL}"/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		awk -F: '$5~/SAP.*Administrator/ {print}' $f | sort -u	
	echo
# TODO sapgw, ...
	echo "/etc/group :"
	f=$(grep -l "^# /etc/group$" "${TEMP}"/"${FIL}"/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		awk -F: '$1=="sapinst" || $1=="sapsys" || $1=="oper" || $1=="saprouter" || $1~/shm/ {print}' $f | sort -u
	echo
	echo "getent_group :"
	f=$(grep -l "^# /usr/bin/getent group$" "${TEMP}"/"${FIL}"/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		awk -F: '$1=="sapinst" || $1=="sapsys" || $1=="oper" || $1=="saprouter" || $1~/shm/ {print}' $f | sort -u
}


function chk_hardware(){

	echo_msgsep "$FUNCNAME"

	echo "machine:"
	FIL="hardware.txt"
	grep -e "system.hardware.*=" -e "system.*.machine.=..s390" ${FIL} | grep -v "system.*video"
	grep "system.firmware.*=" ${FIL}
	grep -e "Manufacturer:.\"[A-Za-z]" -e "Product:.\"" -e "Product.id:.\"" -e "Serial:.\"" ${FIL} | sort -u
	grep -e "str3:.\"../../....\"" -e "str5:.\"[A-Za-z]" ${FIL} |\
		 grep -v "Unknown"
	grep -A4 "^# Virtuali.ation" ${FIL}
	echo
	
	echo "processor:"
	FIL="hardware.txt"
	grep -A16 "^# /usr/bin/lscpu" ${FIL}
	grep -e "^model.name.*:" ${FIL} | sort -u
	grep -A5 "^vendor_id.*:.IBM/S390" ${FIL}
	FIL="boot.txt"
	grep -e "^\[.*Detected.*MHz.processor" -e "^\[.*Brought up.*CPUs" ${FIL}
	echo

	echo "memory:"
	FIL="boot.txt"
	grep "System RAM: " ${FIL} | sort -u

	FIL="basic-health-check.txt"
	grep -A 4 "# /usr/bin/free -k" ${FIL} | tail -3 
	echo

# TODO more specific patterns
#	FIL="hardware.txt"
#	for f in "iLO" "IPMI" "DRAC" "Emulex" "Brocade" "QLogic" "Broadcom" "MPT"; do
#		grep -e ${f} -e "pci.product.=" ${FIL} | sort -u
#	done

	echo "boards:"
	FIL="hardware.txt"
	grep "pci.product.=" ${FIL} |\
		 grep -v "USB.*HCI.*ontroller" | grep -v "PCI.*Port" |\
		 grep -v "I/O Hub" | grep -v "Integrated.Memory.Controller" | sort -u
	grep "scsi.vendor.=" ${FIL} | sort -u
	echo

	echo "disk:"
	FIL="fs-diskio.txt"
	grep -A1 "^Model:" ${FIL} | grep -v "^--$"
	awk '$0~/^Disk..dev.dasd/ {print $2,$3,$4}' ${FIL}
	echo
	echo "storage:"
	FIL="mpio.txt"
	grep "Vendor:.*Model:.*Rev:" ${FIL} | sort -u
	echo
	FIL="hardware.txt"
	for f in "supports.DPO.and.FUA" "support.DPO.or.FUA"; do
		echo -n "${f} : "
		grep -c ${f} ${FIL}
	done
	FIL="messages.txt"
	# shellcheck disable=SC2043
	for f in "disabling.barriers"; do
		echo -n "${f} : "
		grep -c ${f} ${FIL}
	done
	echo
	show_sys_fcfabric
	#awk  -F" " '$8 ~/\/sys\/class\/fc_host\/host/ {print $10}' ${FIL} |\
	echo

	echo "network_card:"
	FIL="network.txt"
	test -z "${NIC_PTRN}" &&\
	NIC_PTRN="
Model:
Device.File:
"
	# TODO better loop
	for f in $NIC_PTRN; do
		grep --binary-file=text -A20 -e "^..: PCI.*Ethernet controller$" -e "HiperSocket.controller$" -e "IBM.Virtual.Ethernet.card" ${FIL} |\
		grep "$f"
	done | sort -u | sed s/Device.File:/"NIC Device File:"/g
	echo	
}


function chk_modules(){

	echo_msgsep "$FUNCNAME"

	# tainted [ 4086.446776]
	grep -e "Modules.linked.in:.*(..)" -e "Modules.linked.in:.*(.)" boot.txt |\
		cut -b16- | sort -u 
	echo

	FIL="boot.txt"
	grep "^\[.*AppArmor:.*initialized" ${FIL}
	echo

	# TODO add Cisco HBA
	# TODO watchdog ppc64le, vmware, xen?
	# TODO check Fusion IO
	test -z "${HARDW_MOD}" &&\
	HARDW_MOD="
ixgbe.ko
netxen_nic.ko
nx_nic.ko
mlx3_en.ko
mlx4_en.ko
mlx5_en.ko
mlx3_ib.ko
mlx4_ib.ko
mlx5_ib.ko
i40e.ko
bfa.ko
lpfc.ko
qlge.ko
qla2xxx.ko
hfcldd.ko
bnx2.ko
bnx.ko
e1000.ko
tg3.ko
igb.ko
be2net.ko
be2iscsi.ko
fnic.ko
enic.ko
vmxnet3.ko
vmw_balloon.ko
vmw_pvscsi.ko
hpsa.ko
cciss.ko
mptbase.ko
dm_multipath.ko
dm_mirror.ko
bonding.ko
hpwdt.ko
hangcheck-timer.ko
deadman.ko
ibmasr.ko
iTCO_wdt.ko
i6300esb.ko
cpu5wdt.ko
advantechwdt.ko
acquirewdt.ko
softdog.ko
vmwatchdog.ko
alim7101_wdt.ko
iomemory-vsl.ko
ibmveth.ko
ibmvnic.ko
ibmvscsi.ko
ibmvfc.ko
lin_tape.ko
"
	FIL="boot.txt"
	for m in INITRD_MODULES MODULES_LOADED_ON_BOOT; do
		grep "^${m}=" ${FIL}
	done
	echo
	for m in $HARDW_MOD; do
		grep "${m}" ${FIL} | sort -u
		n=$(basename "${m}" .ko)
		grep "modprobe ${n}" ${FIL}
	done
	echo
# TODO see below - use split files to records instead of -A9, to find "supported:"
# TODO make ibmvscsi.ko beeing found.
	FIL="modules.txt"
	for m in $HARDW_MOD; do
		grep -A9 "^filename:.*${m}" ${FIL} |\
		 awk '$1=="filename:" || $1=="license:" || $1=="supported:" {print $0}' 
	done
	echo
	for m in $HARDW_MOD; do
		n=$(basename "${m}" .ko)
		grep "options ${n}" ${FIL}
	done
	echo
	FIL="modules.txt"
	split_file ${FIL}
	f=$(grep -l "^# /sbin/lsmod$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -vh "^#==..Co.*==#" $f | sort -u | grep -v "#"
	rm -rf "${TEMP:?}"/${FIL}
	echo	
	echo -n "process on watchdog : "
        awk '$9=="/dev/watchdog" {print $1,$9}' open-files.txt
	echo
	echo
}


function chk_memory(){

	echo_msgsep "$FUNCNAME"

	# TODO page cache, anonymous
	# TODO paging rate, swapspace usage, biggest process?
	# TODO static hugepage
	# TODO tmpfs, w/ or w/o hugepage
	# TODO kernel boot cmdline cgroup..., transparent_hugepage=
	# TODO env.txt ...?
	# TODO memory.txt: slabinfo?
	# TODO cgroups, ulimit
	# TODO memory.txt: /proc/meminfo cached active file and cache inactive M.Jedamzik

	FIL="memory.txt"
	split_file ${FIL}
	echo
	f=$(grep -l "^# /usr/bin/free -k$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -vh "^#==..Co.*==#" $f
	f=$(grep -l "^# /proc/meminfo$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -vh "^#==..Co.*==#" $f
	echo "# /proc/vmstat"
	# TODO what else?
	test -z "${MEM_VMSTAT}" &&\
MEM_VMSTAT="
nr_anon_transparent_hugepages
thp_collapse_alloc_failed
compact_pagemigrate_failed
compact_fail
"
	f=$(grep -l "^# /proc/vmstat$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
	for m in $MEM_VMSTAT; do
		grep "$m" $f
	done
	echo
	awk 'NF==2 && $0~/^numa_miss/ {print $0}' ${FIL}
	echo
	echo "# /sbin/sysctl -a"
	# TODO what else?
	test -z "${MEM_SYSCTL}" &&\
MEM_SYSCTL="
kernel.shmall
kernel.shmmax
kernel.shmmni
vm.overcommit_*
vm.dirty_*
vm.swappiness
vm.pagecache_limit_*
vm.nr_hugepages_*
vm.max_map_count
vm.hugetlb_shm_group
vm.hugepages_treat_as_movable
vm.nr_overcommit_hugepages
vm.lowmem_reserve_ratio
vm.drop_caches
vm.min_free_kbytes
vm.vfs_cache_pressure
vm.memory_failure_early_kill
vm.zone_reclaim_mode
"
	f=$(grep -l "^# /sbin/sysctl -a 2>/dev/null | grep ^vm$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
	for m in $MEM_SYSCTL; do
		grep "$m" $f
	done
	echo
	f=$(grep -l "^# /sys/kernel/mm/transparent_hugepage/enabled$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -vh "^#==..Co.*==#" $f | tr -s "\n"
	f=$(grep -l "^# /sys/kernel/mm/transparent_hugepage/defrag$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -vh "^#==..Co.*==#" $f | tr -s "\n"
	echo	
	FIL="basic-health-check.txt"
	echo "# tmpfs:"
	echo "   Size  Used  Avail  Use%  Mount"
	echo "   Inodes  IUsed  IFree  IUse%  Mount"
	grep "^tmpfs" ${FIL} | tr -s " "
	echo
}


function chk_numa(){

	echo_msgsep "$FUNCNAME"

	FIL="memory.txt"
	split_file ${FIL}
	echo
	f=$(grep -l "^# /usr/bin/numactl --hardware$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -vh "^#==..Co.*==#" $f
# TODO buddyinfo needed?
#	f=$(grep -l "^# /proc/buddyinfo$" "${TEMP}"/${FIL}/xx*)
#	test -z "$f" ||\
#		grep -vh "^#==..Co.*==#" $f
	f=$(grep -l "^# /usr/bin/numastat$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -vh "^#==..Co.*==#" $f
	echo "# /proc/vmstat"
	# TODO what else?
	f=$(grep -l "^# /proc/vmstat$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -e "numa_miss" -e "zone_reclaim_" -e "htlb_buddy_alloc" $f
	echo
	echo "# /sbin/sysctl -a"
	# TODO what else?
	f=$(grep -l "^# /sbin/sysctl -a 2>/dev/null | grep ^vm$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -e "vm.zone_reclaim_mode" -e "vm.numa_zonelist_order"  $f
	echo
}


function chk_services(){

	echo_msgsep "$FUNCNAME"

	FIL="boot.txt"
	# TODO SLES15 what to look for? 
	grep "exits with status" ${FIL} | grep -v "exits with status 0$"
 	grep "Skipped.features:" ${FIL} | sort -u
	echo
	grep "INIT: Entering runlevel:" ${FIL} | sort -u
	echo
	# TODO split systemd.txt "# /bin/ls -alR /etc/systemd/" 
	FIL="systemd.txt"
	test -r "${FIL}" &&\
		awk '$9=="default.target" && $10=="->" {print $9,"->",$11}' ${FIL}
	echo
	FIL="chkconfig.txt"
	if test -r "${FIL}"; then
		for p in "3:on" "3:off"; do
			echo -n "${p} : "
			grep -c "${p}" ${FIL}
		done
	fi
	# TODO split systemd.txt @ "# /bin/systemctl --no-pager --all list-units"
	# TODO single+better awk for SLES12
	FIL="systemd.txt"
	if test -r "${FIL}"; then
		awk '$1~/\.service/ && $2=="enabled" {print}' ${FIL}
		awk '$1~/\.service/ && $2=="disabled" {print}' ${FIL}
	fi
	echo

# TODO precheck_for_sap
# TODO saptune, tuned see chk_saptune
# TODO boot.sapinit with AUTOSTART in instance profile?
# TODO ipmi?
# TODO chrony
	test -z "${SVC_YES}" &&\
	SVC_YES="
boot.clock
boot.devicemapper
boot.lvm
boot.multipath
boot.sapconf
boot.sysstat
boot.sysctl
after.local
multipathd
ntp
sapinit
"
	test -z "${SVC_OFF}" &&\
	SVC_OFF="
boot.dmraid
boot.md
alsasound
hp-asrd
lvm2-lvmetad
mdadmd
oracle
pacemaker
powerd
slpd
smartd
tuned
uuidd
boot.vxfs
qlremote
amf
gab
isisd
llt
vcs
vras-vradmind.sh
vxatd
vxdbdctrl
vxdcli
vxfen
vxnm-vxnetd
vxodm
vxpbx_exchanged
vxrsyncd.sh
vxvm-boot
vxvm-reconfig
vxvm-recover
sfmh-discovery
xprtld
eccmad
"
	FIL="chkconfig.txt"
	if test -r "${FIL}"; then
		echo "service should be on:"
		for s in $SVC_YES; do
			grep "${s}:..on" ${FIL}
			grep "${s}:..off" ${FIL}
			grep "^${s}.*0:.*3:on" ${FIL} | grep -v "boot\."
			grep "^${s}.*0.*3:off" ${FIL} | grep -v "boot\."
		done
		echo
		echo "service should be off:"
		for s in $SVC_OFF; do
			grep "${s}:..off" ${FIL}
			grep "${s}:..on" ${FIL}
			grep "^${s}.*0.*3:off" ${FIL} | grep -v "boot\."
			grep "^${s}.*0.*3:on" ${FIL} | grep -v "boot\."
		done
	fi
	FIL="systemd.txt"
	if test -r "${FIL}"; then
		# TODO SLES12
		echo
	fi
	echo
}


function chk_veritas(){

	echo_msgsep "$FUNCNAME"

	echo "Veritas Storage Foundation and Veritas Cluster Services"
	echo
	grep -e "Modules.linked.in:.*(..)" -e "Modules.linked.in:.*(.)" boot.txt |\
		cut -b16- | sort -u
	echo

	test -z "${VRT_SVC_YES}" &&\
	VRT_SVC_YES="
boot.vxfs
qlremote
amf
isisd
gab
llt
mcelog
vcs
vras-vradmind.sh
vxatd
vxdbdctrl
vxdcli
vxfen
vxnm-vxnetd
vxodm
vxpbx_exchanged
vxrsyncd.sh
vxvm-boot
vxvm-reconfig
vxvm-recover
sfmh-discovery
xprtld
eccmad
"
	FIL="chkconfig.txt"
	if test -r "${FIL}"; then
		for s in $VRT_SVC_YES; do
			grep "${s}:..on" ${FIL}
			grep "${s}:..off" ${FIL}
			grep "^${s}.*0:.*3:on" ${FIL} | grep -v "boot\."
			grep "^${s}.*0.*3:off" ${FIL} | grep -v "boot\."
	done
	fi
	FIL="systemd.txt"
	if test -r "${FIL}"; then
		# TODO SLES12
		echo
	fi
	echo
#
#VRT_MOD="
#vxfen
#gab
#vxodm
#llt
#dmpaa
#vxspec
#vxio
#vxdmp
#vxportal
#fdd
#vxfs
#"
#
	test -z "${VRT_ERR_PTRN}" &&\
	VRT_ERR_PTRN="
"GAB.*failed"
"Symantec.Technical.Support"
"error.*VRT"
"VCS.CRITICAL"
"vxvm.*volume.*not"
"
	LOG="boot.txt messages.txt"
	for e in ${VRT_ERR_PTRN}; do
		echo -n "${e} : "
		# shellcheck disable=SC2086
		for f in ${LOG}; do
			cat "$f"
		done | grep -ic ${e}
	done
	echo
	grep "^#./etc/vx" etc.txt
	grep "^#./etc/opt/VRT" etc.txt
	echo

	FIL="fs-diskio.txt"
	echo -n "VxDMP: "
	grep -c "VxDMP.*/devices/virtual/block/VxDMP" ${FIL}
	echo -n "VxVM:  "
	grep -c "VxVM.*/devices/virtual/block/VxVM" ${FIL}
	echo
}


function chk_network(){

	echo_msgsep "$FUNCNAME"

	test -z "${NETW_SYSCTL}" &&\
	NETW_SYSCTL="
net.core.somaxconn
net.ipv4.tcp_slow_start_after_idle
net.ipv4.ip_local_port_range
net.ipv4.tcp_max_syn_backlog
net.ipv4.tcp_window_scaling
net.core.netdev_max_backlog
net.core.optmem_max
net.core.rmem_default
net.core.rmem_max
net.core.wmem_default
net.core.wmem_max
net.ipv4.conf.all.rp_filter
net.ipv4.tcp_adv_win_scale
net.ipv4.tcp_app_win
net.ipv4.tcp_max_orphans
net.ipv4.tcp_max_tw_buckets
net.ipv4.tcp_mem
net.ipv4.tcp_moderate_rcvbuf
net.ipv4.tcp_no_metrics_save
net.ipv4.tcp_rmem
net.ipv4.tcp_timestamps
net.ipv4.tcp_tw_reuse
net.ipv4.tcp_slow_start_after_idle
net.ipv4.tcp_wmem
net.ipv4.udp_rmem_min
net.ipv4.udp_wmem_min
"
	test -z "${NETW_PTRN}" &&\
	NETW_PTRN="
"Setting.up.hostname"
"Setting.up.loopback.interface"
"eth.*IP.address:.*/"
"bond.*enslaved.interface:"
"bond.*IP.address:.*/"
"
	FIL="boot.txt"
	echo "boot setup:"
	for p in $NETW_PTRN; do
		grep "$p" ${FIL}
	done | sort -u
	echo "last setup:"
	FIL="network.txt"
	for p in $NETW_PTRN; do
		grep --binary-file=text "$p" ${FIL}
	done | sort -u
	echo
	grep --binary-file=text "eth.*\ device:" ${FIL}
	echo

	FIL="network.txt"
	( grep --binary-file=text -A3 "^# /sbin/ethtool bond" ${FIL}
	  grep --binary-file=text -A18 -e "^# /sbin/ethtool hsi" \
		-e "^#.*/sbin/ethtool sit" \
		-e "^#.*/sbin/ethtool eth" ${FIL} ) |\
		 grep -v "^#==\[ " | grep -v "^# /sbin/ethtool" | grep -v "^--$"
	echo
	
	# TODO summary num. NICs, num. "not connected"
	( grep --binary-file=text -A3 "^# /etc/sysconfig/network/routes$" ${FIL} ) |\
		 grep -v "^#==\[ " | grep -v "^# /sbin/ethtool"

	( grep --binary-file=text -A13 "^# /etc/sysconfig/network/ifcfg-bond" ${FIL} |\
		grep -v "=''$"
		echo
	grep --binary-file=text -A21 "^# /proc/net/bonding/bond" ${FIL}

	grep --binary-file=text -A10 \
		-e "^# /etc/sysconfig/network/ifcfg-qeth" \
		-e "^# /etc/sysconfig/network/ifcfg-eth" ${FIL} |\
		 grep -v "=''$" ) |\
		 grep -v "^#==\[ " 
	echo
	# TODO firewall
	echo "iptables:"	
	grep "iptable_" ${FIL}
	echo

	FIL="env.txt"
	split_file ${FIL}
	f=$(grep -l "^# /sbin/sysctl -a$" "${TEMP}"/${FIL}/xx*)
	echo "kernel sysctl settings:"
	# shellcheck disable=SC2086
	test -z "$f" ||\
		for p in $NETW_SYSCTL; do
		grep "$p" $f || echo "$p = NO"
	done
	echo
}


function chk_nameres(){

	echo_msgsep "$FUNCNAME"

	# TODO SLES12
	FIL="network.txt"
	( grep --binary-file=text -A1 "^# /etc/HOSTNAME" ${FIL}
	grep --binary-file=text -A1 "^# /bin/hostname" ${FIL}
	grep --binary-file=text -A1 "^# /sbin/chkconfig nscd" ${FIL} ) |\
		 grep -v "^#==\[ " 
	echo

	split_file ${FIL}
	echo "# /etc/hosts:"
	f=$(grep -l "^# /etc/hosts$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -v "^#" $f | tr -s "\n"
	echo
	echo "# /etc/resolv.conf:"
	f=$(grep -l "^# /etc/resolv.conf$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -v "^#" $f | tr -s "\n"
	echo
#	echo "# /etc/nsswitch.conf:"
#	f=$(grep -l "^# /etc/nsswitch.conf$" "${TEMP}"/${FIL}/xx*)
#	test -z "$f" ||\
#       	grep -v "^#" $f | tr -s "\n"
#	echo
	echo "# /etc/host.conf:"
	f=$(grep -l "^# /etc/host.conf$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
        test -z "$f" ||\
		grep -v "^#" $f | tr -s "\n"

        rm -rf "${TEMP:?}"/${FIL}
	echo

# TODO DNS
}


function chk_timeres(){

	echo_msgsep "$FUNCNAME"

	FIL="ntp.txt"
	echo "software:"
	for p in "^#.rpm.-V" "^#.Verification.Status:"; do
		grep ${p} ${FIL}
	done
	echo
	# TODO SLES12
	# TODO SLES15 chronyd
	# TODO split_file to get complete output of ntpq -p
	split_file ${FIL} 
	echo "service:"
	f=$(grep -l "^# /usr/sbin/ntpq -p$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
        test -z "$f" ||\
		grep -v "^#" $f | tr -s "\n"
	echo
	f=$(grep -l "^# /sbin/chkconfig$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
        test -z "$f" ||\
		grep -v "^#" $f
	f=$(grep -l "^# /bin/systemctl status ntpd.service$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
        test -z "$f" ||\
		grep -v "^#" $f
	echo
	echo "config:"
	echo "# /etc/ntp.conf:"
	f=$(grep -l "^# /etc/ntp.conf$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
        test -z "$f" ||\
		grep -v "^#" $f | tr -s "\n"
	f=$(grep -l "^NTPD_FORCE_SYNC_ON_STARTUP=" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
        test -z "$f" ||\
		grep -v "^#" $f | tr -s "\n"

        rm -rf "${TEMP:?}"/${FIL}
	echo
	grep "Time.zone:" ${FIL}
	echo

	test -z "${TIME_ERR}" &&\
	TIME_ERR="
"synchronized.to.LOCAL"
"ntpd.*time.reset"
"ntpd.*failed"
"
	echo "errors:"
	for p in $TIME_ERR; do
		echo -n "${p} : "
		grep -c "${p}" ${FIL}
	done
	echo

	echo "hwclock:"
	FIL="sysconfig.txt"
	test -z "${TIME_PTRN}" &&\
	TIME_PTRN="
HWCLOCK
SYSTOHC
TIMEZONE
DEFAULT_TIMEZONE
"
	for p in $TIME_PTRN; do
		grep "^${p}" ${FIL} | sort -u
	done
	echo

	FIL="etc.txt"
	grep -A4 "^/etc/adjtime" ${FIL}
	echo

# TODO www.novell.com/support/kb/doc.php?id=7009834 208 days uptime limit
# TODO elevenmin mode, ...
}


function chk_sleha(){

	echo_msgsep "$FUNCNAME"

# TODO cluster md?
	test -z "${CLUSTER_PKG}" &&\
	CLUSTER_PKG="
corosync
cluster-glue
crmsh
ClusterTools2
drbd-heartbeat
drbd-pacemaker
libopenais3
libcorosync4
libglue2
libpacemaker3
ldirectord
pacemaker
pacemaker-mgmt
pacemaker-mgmt-client
resource-agents
sbd
sapstartsrv-resource-agents
sap-suse-cluster-connector
SAPHanaSR
SAPHanaSR-ScaleOut
"
# TODO SLES15 pacemaker-...
	test -z "${CLUSTER_PRCS}" &&\
	CLUSTER_PRCS="
sbd:
/usr/sbin/corosync
/usr/lib64/heartbeat/stonithd
/usr/lib64/heartbeat/cib
/usr/lib64/heartbeat/lrmd
/usr/lib64/heartbeat/attrd
/usr/lib64/heartbeat/pengine
/usr/lib64/heartbeat/crmd
/usr/lib64/heartbeat/mgmtd
"
	FIL="rpm.txt"
	for f in $CLUSTER_PKG; do
		grep "$f" ${FIL}
	done | sort
	echo
	# TODO check /etc/corosync/
	# TODO check ???, see cs_sum_baseconfig, cs_precheck_for_sap
	# TODO check watchdogs, define $WATCHMOD, see cs_disbale_other_watchdog
	# TODO ipmi watchdog
	# hangcheck_timer here, not "-" as in hangcheck-timer.ko
	WATCHMOD="hpwdt hangcheck_timer iTCO ibmasr softdog deadman"
	echo -n "loaded watchdog: "
	for m in $WATCHMOD; do
		awk '$1=="'"${m}"'" {print $1}' modules.txt | sort -u
		awk '$2=="'"${m}"'" {print $0}' boot.txt | sort -u
	done
	echo
        echo -n "process on watchdog: "
        awk '$9=="/dev/watchdog" {print $1,$9}' open-files.txt
        echo
	echo

	FIL="chkconfig.txt"
	if test -r "${FIL}"; then
		split_file ${FIL}
		f=$(grep -l "^# /sbin/chkconfig.--list$" "${TEMP}"/${FIL}/xx*)
		echo "/sbin/chkconfig --list:"
		# shellcheck disable=SC2086
		test -z "$f" ||\
			grep -e "^openais.*0:" -e "^hp-asrd.*0:" $f
		echo
		f=$(grep -l "^/etc/init.d/rc3.d:$" "${TEMP}"/${FIL}/xx*)
        	echo "/etc/init.d/rc3.d:"
		# shellcheck disable=SC2086
		test -z "$f" ||\
			grep -e "K..openais" -e "S..hp-asrd" $f
		echo
		f=$(grep -l "^/etc/init.d/rc5.d:$" "${TEMP}"/${FIL}/xx*)
        	echo "/etc/init.d/rc5.d:"
		# shellcheck disable=SC2086
		test -z "$f" ||\
			grep -e "K..openais" -e "S..hp-asrd" $f
		echo
	fi
	FIL="systemd.txt"
	if test -r "${FIL}"; then
		# TODO SLES12
		echo
	fi

	FIL="sysconfig.txt"
        split_file ${FIL}
        f=$(grep -l "^# /etc/sysconfig/openais$" "${TEMP}"/${FIL}/xx*)
	echo "/etc/sysconfig/openais:"
	# shellcheck disable=SC2086
	test -z "$f" || grep -v "^#" $f | tr -s "\n"
	echo
        f=$(grep -l "^# /etc/sysconfig/pacemaker$" "${TEMP}"/${FIL}/xx*)
	echo "/etc/sysconfig/pacemaker:"
	# shellcheck disable=SC2086
	test -z "$f" || grep -v "^#" $f | tr -s "\n"
	echo	
	f=$(grep -l "^# /etc/sysconfig/sbd$" "${TEMP}"/${FIL}/xx*)
	echo "/etc/sysconfig/sbd:"
	# shellcheck disable=SC2086
	test -z "$f" || grep -v "^#" $f | tr -s "\n"
	echo

	FIL="ha.txt"
        split_file ${FIL}
	f=$(grep -l "^# /usr/sbin/sbd.-d.*dump$" "${TEMP}"/${FIL}/xx*)
	echo "sbd -d /dev/... dump:"
	# shellcheck disable=SC2086
	test -z "$f" || grep -v "^#" $f
	echo
	f=$(grep -l "^# /usr/sbin/sbd.-d.*list$" "${TEMP}"/${FIL}/xx*)
	echo "sbd -d /dev/... list:"
	# shellcheck disable=SC2086
	test -z "$f" || grep -v "^#" $f
	echo
        f=$(grep -l "^# /etc/corosync/corosync.conf$" "${TEMP}"/${FIL}/xx*)
	echo "/etc/corosync/corosync.conf:"
	# shellcheck disable=SC2086
	test -z "$f" || grep -v "^#" $f
	echo
        f=$(grep -l "^# /usr/sbin/corosync-cfgtool -s$" "${TEMP}"/${FIL}/xx*)
	echo "/usr/sbin/corosync-cfgtool -s:"
	# shellcheck disable=SC2086
	test -z "$f" || grep -v "^#" $f
	echo
	f=$(grep -l "^# /usr/sbin/sbd.-d.*dump$" "${TEMP}"/${FIL}/xx*)
        f=$(grep -l "^# /usr/sbin/crm_mon.-r.-1$" "${TEMP}"/${FIL}/xx*)
	echo "crm_mon -r -1:"
	# shellcheck disable=SC2086
	test -z "$f" || grep -v "^#" $f
	echo
        f=$(grep -l "^# /var/lib/heartbeat/crm/cib.xml$" "${TEMP}"/${FIL}/xx*)
	echo "/var/lib/heartbeat/crm/cib.xml:"
	# shellcheck disable=SC2086
	test -z "$f" || grep -v "^#" $f
        f=$(grep -l "^# /var/lib/pacemaker/cib/cib.xml$" "${TEMP}"/${FIL}/xx*)
	echo "/var/lib/pacemaker/cib/cib.xml:"
	# shellcheck disable=SC2086
	test -z "$f" || grep -v "^#" $f
        f=$(grep -l "^# /usr/sbin/crm configure show$" "${TEMP}"/${FIL}/xx*)
	echo "/usr/sbin/crm configure show:"
	# shellcheck disable=SC2086
	test -z "$f" || grep -v "^#" $f
	echo
	# TODO crm configure show?

	FIL="basic-health-check.txt"
	echo "cluster processes running:"
	for p in $CLUSTER_PRCS; do 
		awk '$10=="'"${p}"'" {print $0}' ${FIL}
	done
	echo

	FIL="lvm.txt"
        split_file ${FIL}
        echo "/etc/lvm/lvm.conf:"
        f=$(grep -l "# /etc/lvm/lvm.conf" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
        test -z "$f" || grep -vh "^#" $f | grep "filter"
        echo
	FIL="fs-softraid.txt"
        split_file ${FIL}
        echo "/etc/mdadm.conf:"
        f=$(grep -l "# /etc/mdadm.conf" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
        test -z "$f" || grep -vh "^#" $f # | grep "/dev/null"
        echo

	# TODO /proc/sys/net/ipv4/tcp_retries2 : 2 for NW Enqueue Replication
}


function chk_xenhost(){

	echo_msgsep "$FUNCNAME"

	# TODO what is really needed?
	FIL="xen.txt"
        split_file ${FIL}
	echo
	echo "# /boot/grub/menu.lst"
	f=$(grep -l "# /boot/grub/menu.lst" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" || grep -e "kernel /boot/xen.gz" -e "module /boot/vmlinuz.*-xen" -e "module /boot/initrd.*-xen" $f
	echo
	# TODO more xen specific RPMs?
	grep -e "vmdp" -e "libvirt" rpm.txt
	echo
	f=$(grep -l "# /etc/xen/xend-config.sxp" "${TEMP}"/${FIL}/xx*)
	echo "# /etc/xen/xend-config.sxp"
	# shellcheck disable=SC2086
	test -z "$f" || grep -vh "^#" $f
	echo
	f=$(grep -l "# /etc/sysconfig/xendomains" "${TEMP}"/${FIL}/xx*)
	echo "# /etc/sysconfig/xendomains"
	# shellcheck disable=SC2086
	test -z "$f" || grep -vh "^#" $f
	# TODO xm info, xm list
	# TODO network, LUNs
	# TODO Xen errors in syslog

	FIL="modules.txt"
        split_file ${FIL}
	# TODO what filenames?
	echo "# /etc/modprobe.d/xen-blkbk.conf"
	f=$(grep -l "# /etc/modprobe.d/xen-blkbk.conf" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" || grep -vh "^#" $f
	echo
	echo "# /sbin/modinfo blkbk"
	f=$(grep -l "# /sbin/modinfo blkbk" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" || grep -vh "^#" $f
	echo
	# TODO netbk some options only from /etc/modules.d/*; what filenames?
	echo "# /etc/modprobe.d/xen-netbk.conf"
	f=$(grep -l "# /etc/modprobe.d/xen-netbk.conf" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" || grep -vh "^#" $f
	echo
	echo "# /sbin/modinfo netbk"
	f=$(grep -l "# /sbin/modinfo netbk" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" || grep -vh "^#" $f
	echo
	# TODO blkbk "max_ring_page_order=0..4" from /etc/modules.d/*; what filenames?
	echo "# /etc/modprobe.d/iomemory-vsl.conf"
	f=$(grep -l "# /etc/modprobe.d/iomemory-vsl.conf" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" || grep -vh "^#" $f
	echo	
	echo "# /sbin/modinfo iomemory_vsl (propietary Fusion-IO)"
	f=$(grep -l "# /sbin/modinfo iomemory_vsl" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" || grep -vh "^#" $f
	echo
	echo "# grep for options [netbk|blkbk|iomemory_vsl]:"
	grep -e "options netbk" -e "options blkbk" -e "options iomemory_vsl" ${FIL}
	echo
}


function chk_powervm(){

	echo_msgsep "$FUNCNAME"

	# TODO IBM power tools
	test -z "${PWVM_PKG}" &&\
	PWVM_PKG="
glibc2
kernel-default
ntp-4
DynamicRM
IBMinvscout
TIV-TSMERP-HANA
TIVsm-API64
TIVsm-BA
gskcrypt64
gskssl64
ibm-power-managed-sles12
ibmPMLinux
iprutils
librtas1
librtas2
lpar2rrd-agent
powerpc-utils
powerpc-utils-python
ppc64-diag
"
	# TODO systemd.txt services
	test -z "${PWVM_SVC_YES}" &&\
	PWVM_SVC_YES="
ntp
srcmstr
iprdump
iprinit
iprupdate
"

	# TODO mpio modules, e.g. alua?
	# TODO modules.txt /sys/module/ibmvfc/parameters/
	# TODO modules.txt /sys/module/ibmveth/parameters/ 
	test -z "${PWVM_MODUL}" &&\
	PWVM_MODUL="
ibmveth
ivmvnic
ibmvscsi
ibmvfc
ibmpowernv
ibmpex
ibmaem
lin_tape
rpaphp
scsi_dh_alua
softdog
"
	# TODO which files?
	test -z "${PWVM_ETC}" &&\
	PWVM_ETC="
"/etc/ntp.conf"
"/etc/multipath.conf"
"/etc/sysconfig/network/ifcfg-eth0"
"/etc/sysconfig/network/ifcfg-eth1"
"

	echo "- packages installed:"
	FIL="rpm.txt"
	for p in $PWVM_PKG; do
		echo -n "$p : "
		# TODO awk only
		grep "$p" ${FIL} | awk '{print $1}'
	done
	echo
	echo
	echo "- modules loaded:"
	FIL="modules.txt"
	for p in $PWVM_MODUL; do
		echo -n "$p : "
		grep "$p" ${FIL}	
	done
	echo
	FIL="chkconfig.txt"
	if test -r "${FIL}"; then
		echo "- service should be on:"
		for s in $PWVM_SVC_YES; do
			grep "${s}:..on" ${FIL}
			grep "${s}:..off" ${FIL}
			grep "^${s}.*0:.*3:on" ${FIL} | grep -v "boot\."
			grep "^${s}.*0.*3:off" ${FIL} | grep -v "boot\."
		done
		echo
		echo "- service should be off:"
		for s in $PWVM_SVC_OFF; do
			grep "${s}:..off" ${FIL}
			grep "${s}:..on" ${FIL}
			grep "^${s}.*0.*3:off" ${FIL} | grep -v "boot\."
			grep "^${s}.*0.*3:on" ${FIL} | grep -v "boot\."
		done
	fi
	FIL="systemd.txt"
	if test -r "${FIL}"; then
		# TODO SLES12
		echo
	fi
	echo
	echo
	echo "- config files:"	
	FIL="etc.txt"
	split_file ${FIL}
	for p in $PWVM_ETC; do
		echo
		echo "$p :"
		f=$(grep -l "^# ${p}$" "${TEMP}"/${FIL}/xx*)
		test -z "$f" && continue
		# shellcheck disable=SC2086
		test -r $f && grep -v "^#" $f | tr -s "\n"
	done
	FIL="network.txt"
	split_file ${FIL}
	for p in $PWVM_ETC; do
		echo
		echo "$p :"
		f=$(grep -l "^# ${p}$" "${TEMP}"/${FIL}/xx*)
		test -z "$f" && continue
		# shellcheck disable=SC2086
		test -r $f && grep -v "^#" $f | tr -s "\n"
	done
	echo
	echo
	#show_sys_scsi
	echo
}


function chk_vmware(){

	echo_msgsep "$FUNCNAME"

	# http://www.vmware.com/download/packages.html
	# http://packages.vmware.com/tools
	# TODO what is really needed?
	# TODO chrony
	test -z "${VMW_PKG}" &&\
	VMW_PKG="
ntp-4
vmware-tools-core
vmware-tools-esx
vmware-tools-esx-kmods-default
vmware-tools-esx-nox
vmware-tools-guestlib
vmware-tools-guestsdk
vmware-tools-hgfs
vmware-tools-libraries-nox
vmware-tools-libraries-x
vmware-tools-plugins-autoUpgrade
vmware-tools-plugins-deployPkg
vmware-tools-plugins-desktopEvents
vmware-tools-plugins-dndcp
vmware-tools-plugins-guestInfo
vmware-tools-plugins-hgfsServer
vmware-tools-plugins-powerOps
vmware-tools-plugins-resolutionSet
vmware-tools-plugins-timeSync
vmware-tools-plugins-unity
vmware-tools-plugins-vix
vmware-tools-plugins-vmbackup
vmware-tools-pvscsi-common
vmware-tools-pvscsi-kmp-default
vmware-tools-services
vmware-tools-thinprint
vmware-tools-user
vmware-tools-vmblock-common
vmware-tools-vmblock-kmp-default
vmware-tools-vmci-common
vmware-tools-vmci-kmp-default
vmware-tools-vmhgfs-common
vmware-tools-vmhgfs-kmp-default
vmware-tools-vmmemctl-common
vmware-tools-vmmemctl-kmp-default
vmware-tools-vmsync-common
vmware-tools-vmsync-kmp-default
vmware-tools-vmxnet-common
vmware-tools-vmxnet-kmp-default
vmware-tools-vmxnet3-common
vmware-tools-vmxnet3-kmp-default
vmware-tools-vsock-common
vmware-tools-vsock-kmp-default
vmware-tools-xorg-drv-display-
vmware-tools-xorg-drv-mouse
"
	test -z "${VMW_MODUL}" &&\
	VMW_MODUL="
vmxnet3
vmmouse
vmw_pvscsi
vmw_balloon
mptbase
"
	# TODO vmware tools services?
	# TODO chrony
	test -z "${VMW_SVC_YES}" &&\
	VMW_SVC_YES="
boot.clock
boot.sysstat
boot.sysctl
ntp
vmware-tools-services
"
	test -z "${VMW_SVC_OFF}" &&\
	VMW_SVC_OFF="
alsasound
ipmi
irqbalance
joystick
libvirtd
mcelog
microcode.ctl
powerd
smartd
xend
xendomains
"
	test -z "${VMW_ETC}" &&\
	VMW_ETC="
"/etc/ntp.conf"
"/etc/vmware-tools/tools.conf"
"/etc/vmware-tools/plugins"
"/etc/pam.d/vmtoolsd"
"/etc/ld.so.conf.d/vmware-tools-libraries.conf"
"
	# TODO  vmware user /usr/lib/vmware-tools/moduleScripts/user/vmware-config.pl --toConfigure=user

	echo "- packages installed:"
	FIL="rpm.txt"
	for p in $VMW_PKG; do
		echo -n "$p : "
		# TODO awk only
		grep "$p" ${FIL} | awk '{print $1}'
	done
	echo
	echo
	echo "- modules loaded:"
	FIL="modules.txt"
	for p in $VMW_MODUL; do
		echo -n "$p : "
		grep "$p" ${FIL}	
	done
	echo
	# TODO common function for services, see chk_services, chk_sap, chk_oracle
	echo "- service should be on:"
	FIL="chkconfig.txt"
	if test -r "${FIL}"; then
		for s in $VMW_SVC_YES; do
			grep "${s}:..on" ${FIL}
			grep "${s}:..off" ${FIL}
			grep "^${s}.*0:.*3:on" ${FIL} | grep -v "boot\."
			grep "^${s}.*0.*3:off" ${FIL} | grep -v "boot\."
		done
		echo
		echo "- service should be off:"
		for s in $VMW_SVC_OFF; do
			grep "${s}:..off" ${FIL}
			grep "${s}:..on" ${FIL}
			grep "^${s}.*0.*3:off" ${FIL} | grep -v "boot\."
			grep "^${s}.*0.*3:on" ${FIL} | grep -v "boot\."
		done
	fi
	FIL="systemd.txt"
	if test -r "${FIL}"; then
		# TODO SLES12
		echo
	fi
	echo
	echo -n "- vmtoolsd running: "
	FIL="basic-health-check.txt"
	awk '$10~/vmtoolsd/ {print $10}' ${FIL}
	echo
	echo "- config files:"	
	FIL="etc.txt"
	split_file ${FIL}
	for p in $VMW_ETC; do
		echo
		echo "$p :"
		f=$(grep -l "^# ${p}$" "${TEMP}"/${FIL}/xx*)
		test -z "$f" && continue
		# shellcheck disable=SC2086
		test -r $f && grep -v "^#" $f | tr -s "\n"
	done
	echo
	show_sys_scsi
	echo
}


function chk_saptune2(){

	echo_msgsep "$FUNCNAME"

	FIL=rpm.txt
	echo "RPM OK:"
	grep saptune-2 $FIL
	echo "RPM NO:"
	grep -e sapconf -e saptune-1 $FIL
	rm -rf "${TEMP:?}"/${FIL}
	echo

	FIL=systemd.txt
	# TODO awk
	echo "service OK:"
	# shellcheck disable=SC2062
	grep -e ^tuned.service.*enabled -e tuned.service.*running $FIL |\
		awk '{print $1,$2,$3,$4}'
	echo service NO:
	# shellcheck disable=SC2062
	grep -e ^tuned.service.*disabled -e tuned.service.*dead $FIL |\
		awk '{print $1,$2,$3,$4}'
	rm -rf "${TEMP:?}"/${FIL}
	echo

	FIL=tuned.txt
	split_file ${FIL}
	f=$(grep -l "^# /bin/systemctl status tuned.service$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -e "Loaded:" -e "Active:" $f

	f=$(grep -l "^# /usr/sbin/tuned-adm active$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -v "^#" $f

	f=$(grep -l "^# /usr/lib/tuned/saptune/tuned.conf$" "${TEMP}"/${FIL}/xx*)
	test -z "$f" ||\
		echo "/usr/lib/tuned/saptune/tuned.conf:"
	# shellcheck disable=SC2086
	grep -v "^#" $f

	# TODO make this work
#	f=$(grep -l "^# /var/log/tuned/tuned.log$" "${TEMP}"/${FIL}/xx*)
#	test -z "$f" ||\
#		echo "/var/log/tuned/tuned.log:"
#		grep -v "^#" $f #| grep -v "^[1-9].*[0-9][0-9][0-9].INFO"

	# TODO sap-hana.conf
	rm -rf "${TEMP:?}"/${FIL}
	echo

	FIL=sysconfig.txt
	split_file ${FIL}
	f=$(grep -l "^# /etc/sysconfig/saptune$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -v "^#" $f
	rm -rf "${TEMP:?}"/${FIL}
	echo

	FIL=etc.txt
	split_file ${FIL}
	# TODO loop for below 6 files
	f=$(grep -l "^# /etc/security/limits.d/saptune-\@sapsys-nofile-hard.conf$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -v "#==\[" $f
	f=$(grep -l "^# /etc/security/limits.d/saptune-\@sapsys-nofile-soft.conf$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -v "#==\[" $f
	f=$(grep -l "^# /etc/security/limits.d/saptune-\@dba-nofile-hard.conf$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -v "#==\[" $f
	f=$(grep -l "^# /etc/security/limits.d/saptune-\@dba-nofile-soft.conf$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -v "#==\[" $f
	f=$(grep -l "^# /etc/security/limits.d/saptune-\@sdba-nofile-hard.conf$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -v "#==\[" $f
	f=$(grep -l "^# /etc/security/limits.d/saptune-\@sdba-nofile-soft.conf$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -v "#==\[" $f

	f=$(grep -l "^# /etc/sysconfig/saptune$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -v "^#" $f | sort -u
	f=$(grep -l "^# /etc/sysctl.d/99-sysctl.conf$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -v "#==\[" $f
	# TODO SLE12 only: saptune-UserTasksMax.conf
	f=$(grep -l "^# /etc/systemd/logind.conf.d/saptune-UserTasksMax.conf$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -v "#==\[" $f
	rm -rf "${TEMP:?}"/${FIL}
	echo
	echo "file OK:"
	grep -e "/etc/saptune/" -e "/saptune-" $FIL | sort -u | tr -d "#"
	echo "file NO:"
	FIL=sysconfig.txt
	grep -e "/etc/sysconfig/sapconf$" $FIL | tr -d "#"
	echo "file NO:"
	grep "/etc/sysconfig/sapnote-" $FIL | tr -d "#"
	echo
}
 

function chk_sap(){

	echo_msgsep "$FUNCNAME"

        test -z "${SAP_SAP_PKG_NOT}" &&\
        SAP_SAP_PKG_NOT="
ulimit
"
	# TODO check /etc/sysconfig/sapconf
	echo "sysconfig :"
	FIL=etc.txt
	split_file ${FIL}
	f=$(grep -l "^# /etc/sysconfig/sapconf$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -v "^#" $f | sort -u
	echo
	# TODO check ???, see sum_base_config, precheck_for_sap
	grep -e "^@sapsys" -e "^@sdba" crash.txt
	echo
	echo "proc :"
	FIL=proc.txt
	split_file ${FIL}
	f=$(grep -l "^# /proc/net/unix$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		awk '$8~/sap/ || $8~/oracle/ {print $8}' $f | sort -u
	echo
	FIL=pam.txt
	split_file ${FIL}
	show_pam_sap
	echo
	echo "proc :"
	FIL=memory.txt
	split_file ${FIL}
	f=$(grep -l "^# /usr/bin/pmap [1-9]" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		awk '$2=="sapstartsrv" || $2=="sapstart" || $2~/ms\.sap/ || $2~/en\.sap/ || $2~/ig\.sap/ || $2=="igsmux_mt" || $2=="igspw_mt" || $2=="gwrd" || $2=="icman" || $2~/jc\.sap/ || $2=="jstart" {print}' $f | sort -u
		# shellcheck disable=SC2086
		awk '$8~/sapstart/ || $8~/disp\+work/ || $8~/igswd_mt/ || $8~/igspw_mt/ || $8~/igsmux_mt/ || $8~/gwrd/ || $8~/icman/ || $8~/dboraslib/ || $8~/libaio/ || $8~/libclntsh.so/ || $8~/dw_/ || $8~/jstart/ {print $8}' $f | sort -u
	echo

# TODO 1310037   SUSE LINUX Enterprise Server 11: Installation notes?
# TODO SAP Application Server Base, Minimal System (Appliances), X Window System, Print Server, C/C++ Compiler and Tools
# TODO no 32bit pattern 
# TODO no /etc/hosts 127.0.0.2   <hostname>.<domain> <hostname>
# TODO 1275776	Linux: Preparing SLES for SAP environments
# TODO /usr/sbin/uuidd         uuidd:uuidd     6755, chkstat --set /etc/permissions.local
# TODO 1557506 - Linux paging improvements
# The recommended value is 1/16 (~6%) of the amount of RAM, but not less than 512 MByte and not more than 4 GByte.
#<  8 GB:  512  (remommended min. limit)
#  8 GB:  512  (=  8 * 1024 MB / 16)
#  16 GB: 1024  (= 16 * 1024 MB / 16)
#  32 GB: 2048  (= 32 * 1024 MB / 16)
#  64 GB: 4096  (= 64 * 1024 MB / 16)
#> 64 GB: 4096  (recommended max. limit)
#vm.pagecache_limit_ignore_dirty
# If there is a lot of local writes and it is OK to throttle them by limiting the writeback caching, we recommended that you set the value to 0. If writing mainly happens to NFS filesystems, the default 1 should be left untouched.(A value of 2 would be a middle ground, not limiting local write back caching as much, but potentially resulting in some paging.)

	# TODO -32bit?
	echo "RPMs forbidden by SAP:"
	for f in $SAP_SAP_PKG_NOT; do
		echo -n "$f : "
		grep -v -- "-32bit" ${FIL} | grep "^${f}-[0-9]" || echo "NO"
	done | awk '{print $1,":",$3}' | sort
	echo
}


function chk_saphana(){

	echo_msgsep "$FUNCNAME"

# TODO current SAP notes 19xxxx and  2xxxx for HANA 2.0
# TODO chk_sap for 1310037   SUSE LINUX Enterprise Server 11: Installation notes?
# TODO chk_sap for 1275776   prepare SLES ...?
# 1514967   SAP HANA: Central Note
# 1523337   SAP In-Memory Database 1.0: Central Note
# 1944799   SAP HANA Guidelines for SLES Operating System Installation
# 1824819   SAP HANA DB: Recommended OS settings for SLES 11 / SLES for SAP Applications 11 SP2
# 1954788   SAP HANA DB: Recommended OS settings for SLES 11 / SLES for SAP Applications 11 SP3
# 1855805   Recommended SLES 11 packages for HANA support on OS level
# 1890444   Slow HANA system due to CPU power save mode
# 1867783   XFS Data Inconsistency Bug with SLES 11 SP2
# 1888072   SAP HANA DB: Indexserver crash in __strcmp_sse42
# TODO 1846872   "No space left on device" error reported from HANA
# TODO 1681360 - Support for SAP software in Linux / Fusion-IO
# 784391	SAP support terms and 3rd-party Linux kernel drivers
# 171356	SAP software on Linux: General information
# TID 7011982 cstate
# TODO krb5-32bit needs 32bit pattern 
# TODO Fusion IO: 
#	IBM, Dell: modprobe iomemory-vsl
#		/etc/modprobe.d/iomemory-vsl.conf:  allow_unsupported_modules 1
#	Cisco: messages:  fioinf Fusion-io ioDrive 320GB 
# TODO HANA does not allow dash "-" in hostnames
# TODO /usr/sap 50GB, <hana-data> 3*RAM, <hana-shared> 1*RAM, <hana-log> 1*RAM max. 512GB 
# TODO 10gb/s NIC for system replication
# TODO MTU=9000 for system replication
# TODO udev.txt: # /usr/lib/udev/rules.d/60-ssd-scheduler.rules -> deadline

	test -z "${HANA_SAP_BOOTARG}" &&\
	HANA_SAP_BOOTARG="
intel_idle.max_cstate=1
processor.max_cstate=1
transparent_hugepage=never
numa_balancing=disable
"
# TODO nospectre_v2 nopti?
	test -z "${HANA_SUSE_BOOTARG}" &&\
	HANA_SUSE_BOOTARG="
elevator=noop
cgroup_disable=memory
"
# TODO look for elevator also in udev.txt
# TODO should sysctl go to chk_sap?
	test -z "${HANA_SAP_SYSCTL}" &&\
	HANA_SAP_SYSCTL="
vm.pagecache_limit_mb
vm.pagecache_limit_ignore_dirty
vm.overcommit_memory
vm.swappiness
vm.max_map_count
vm.memory_failure_early_kill
fs.aio-max-nr
fs.file-max
kernel.sem
kernel.shmall
kernel.shmmax
kernel.shmmni
kernel.numa_balancing
net.core.somaxconn
net.ipv4.tcp_slow_start_after_idle
net.ipv4.ip_local_port_range
net.ipv4.tcp_max_syn_backlog
net.ipv4.tcp_window_scaling
"
	test -z "${HANA_SUSE_SYSCTL}" &&\
	HANA_SUSE_SYSCTL="
vm.dirty_bytes
vm.dirty_background_bytes
"
	test -z "${HANA_VERS_MIN}" &&\
	HANA_VERS_MIN="
kernel-ppc64le:4.4.156-94.57
kernel-default:4.4.156-94.57
kernel-xen:4.4.156-94.57
glibc2:2.22-62.16
libgcc_s1:6.2.1
libstdc++6:6.2.1
libuuid1=2.25-22
sapinit-systemd-compat=1.0-2.1
saptune=2.0.3-3.11.1
util-linux=2.25-22
xfsprogs:3.1.8-0.5.1
"
	# TODO chrony
	test -z "${HANA_SAP_PKG_YES}" &&\
	HANA_SAP_PKG_YES="
hal
gtk2
iptraf
java-1_7_0-ibm
libicu
mozilla-xulrunner192
ntp
sudo
syslog-ng
tcsh
libssh2-1
libstdc++6
libgcc_s1
uuidd
expect
autoyast2-installation
yast2-ncurses
bing
bonnie
cairo
findutils-locate
graphviz
iptraf
krb5-32bit
krb5-client
nfs-client
sensors
"
	test -z "${HANA_SAP_PKG_NOT}" &&\
	HANA_SAP_PKG_NOT="
ulimit
"
	# TODO chrony
	test -z "${HANA_SAP_SVC_YES}" &&\
	HANA_SAP_SVC_YES="
boot.sysstat
boot.sapconf
boot.clock
ntp
sapinit
"
	test -z "${HANA_SAP_SVC_OFF}" &&\
	HANA_SAP_SVC_OFF="
boot.kdump
"
	test -z "${HANA_SUSE_PTRN}" &&\
	HANA_SUSE_PTRN="
base
x11
Basis-Devel
sap_hana
ha_sles
"
# TODO Fusion IO driver RPMs per vendor? (IBM, Dell, HP are same)
	test -z "${HANA_SUSE_PKG_YES}" &&\
HANA_SUSE_PKG_YES="
rear1172a
sysstat
supportutils
sapconf
saptune
xfsprogs
xfsdump
fio-util
fio-common
fio-sysvinit
libvsl
iomemory-vsl
fio-firmware-ioaccelerator
fio-firmware-highiops
SAPHanaSR
SAPHanaSR-doc
SAPHanaSR-ScaleOut
SAPHanaSR-ScaleOut-doc
HANA-Firewall
"

	#TODO HANA_SAP_SYSCTL HANA_SUSE_SYSCTL HANA_SAP_SVC_YES HANA_SAP_SVC_OFF HANA_SUSE_PTRN
	# TODO boot.local: performance > /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor
	# TODO boot.local: never > /sys/kernel/mm/transparent_hugepage/enabled

	FIL="rpm.txt"
	echo "RPM with specific version recommended by SAP:"
	for f in $HANA_VERS_MIN; do
		FULL=$(echo "$f" | awk -F: '{print $1"-"$2}')
		BASE=$(echo "$f" | awk -F: '{print $1}')
		# TODO xfsprogs-3.1.8-0.5.1 : xfsprogs-3.1.8-0.5.1 Mon Aug
		echo -n "$FULL : "
		q="$FULL"
		# TODO do search, if package is -32bit
		grep -v -- "-32bit" ${FIL} | grep "^${FULL}" ||\
		 	q=$(	p=$(grep -v -- "-32bit" ${FIL} |\
					grep  "^${BASE}-[0-9]")
				echo "NO but $p"
			)
		echo "${q}"
	done | awk '{print $1,":",$3,$4,$5}' | sort
	echo
	FIL="boot.txt"
	grep -A1 "^# /bin/uname -a" ${FIL}
	echo

	FIL="rpm.txt"
	echo "RPMs recommended by SAP:"
	for f in $HANA_SAP_PKG_YES; do
		echo -n "$f : "
		# TODO do search, if package is -32bit
		grep -v -- "-32bit" ${FIL} | grep "^${f}-[0-9]" || echo "NO"
	done | awk '{print $1,":",$3}' | sort
	echo
	echo "RPMs recommended by SUSE:"
	for f in $HANA_SUSE_PKG_YES; do
		echo -n "$f : "
		# TODO do search, if package is -32bit
		grep -v -- "-32bit" ${FIL} | grep "^${f}-[0-9]" || echo "NO"
	done | awk '{print $1,":",$3}' | sort
	echo
	# TODO HANA_SAP_PKG_NOT. -32bit?
	echo "RPMs forbidden by SAP:"
	for f in $HANA_SAP_PKG_NOT; do
		echo -n "$f : "
		grep -v -- "-32bit" ${FIL} | grep "^${f}-[0-9]" || echo "NO"
	done | awk '{print $1,":",$3}' | sort
	echo

	FIL="boot.txt"
	KRNLCMD=$(grep -A1 "^# /proc/cmdline" ${FIL} | tail -1)
	echo "SAP recommended /proc/cmdline args:"
	# TODO shorter output
	for f in $HANA_SAP_BOOTARG; do
		echo -n "$f : "
		p="NO"
		q=$( echo "$KRNLCMD" | grep "$f" ) && p=$q
		echo "$p"
	done
	echo
	echo "SUSE recommended /proc/cmdline args:"
	# TODO shorter output
	for f in $HANA_SUSE_BOOTARG; do
		echo -n "$f : "
		p="NO"
		q=$( echo "$KRNLCMD" | grep "$f" ) && p=$q
		echo "$p"
	done
	echo
	echo "Current /proc/cmdline:"
	echo "$KRNLCMD"
	echo
	
	FIL="sysconfig.txt"
	split_file ${FIL}
	hana_defapd=$(echo "${HANA_SAP_BOOTARG}" "${HANA_SUSE_BOOTARG}" | tr -d "\n")
	echo "SAP+SUSE recommended DEFAULT_APPEND: $hana_defapd"
	echo "# /etc/sysconfig/bootloader:"
	f=$(grep -l "^# /etc/sysconfig/bootloader$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep DEFAULT_APPEND $f
	echo
	rm -rf "${TEMP:?}"/${FIL}

	# TODO HANA_SAP_SYSCTL, HANA_SUSE_SYSCTL as SAP_SYSCTL in chk_sap? 
	FIL="etc.txt"
	split_file ${FIL}
	f=$(grep -l "^# /etc/sysctl.conf$" "${TEMP}"/${FIL}/xx*)
	echo "SAP recommended sysctl:"
	# shellcheck disable=SC2086
	test -z "$f" ||\
		for p in $HANA_SAP_SYSCTL; do
			echo -n "$p : "
			grep "$p" $f || echo "NO"
		done
	echo
	echo "SUSE recommended sysctl:"
	# shellcheck disable=SC2086
	test -z "$f" ||\
		for p in $HANA_SUSE_SYSCTL; do
			echo -n "$p : "
			grep "$p" $f || echo "NO"
		done
	echo
	rm -rf "${TEMP:?}"/${FIL}

	# TODO /boot/grub/menu.lst
	
	FIL="boot.txt"
	split_file ${FIL}
	# TODO
	echo
	grep -A1 "^# /sys/kernel/mm/transparent_hugepage/enabled" memory.txt |\
		tr -d "#"
	echo "SAP recommended boot.local: echo never > /sys/kernel/mm/transparent_hugepage/enabled"
	echo "# /etc/init.d/boot.local:"
	F=$(grep -l "# /etc/init.d/boot.local" "${TEMP}"/${FIL}/xx* | tr -d ":#" | awk '{print $1}')
	# shellcheck disable=SC2086
	test -z "$F" ||\
		grep -v "^#" $F

	FIL="sysfs.txt"
	echo "SAP recommeded nr_anon_transparent_hugepages: 0 , nr_hugepages: 0"
	grep "nr_anon_transparent_hugepages " ${FIL} | grep -v -- "--" 
	grep -A1 "# /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" ${FIL} | grep -v -- "--"
	
# TODO fix it: csplit runs on 100% CPU for ever
#	split_file ${FIL}
#
#	echo "SAP recommeded nr_anon_transparent_hugepages: 0 , nr_hugepages: 0"
#	# TODO n NUMA nodes
#	F=$(grep -l "# /sys/devices/system/node/node0/vmstat" "${TEMP}"/${FIL}/xx* | tr -d ":#" | awk '{print $1}')
#		test -z "$F" ||\
#			grep "nr_anon_transparent_hugepages" $F
#	F=$(grep -l "# /sys/devices/system/node/node1/vmstat" "${TEMP}"/${FIL}/xx* | tr -d ":#" | awk '{print $1}')
#		test -z "$F" ||\
#			grep "nr_anon_transparent_hugepages" $F
#	F=$(grep -l "# /sys/devices/system/node/node2/vmstat" "${TEMP}"/${FIL}/xx* | tr -d ":#" | awk '{print $1}')
#		test -z "$F" ||\
#			grep "nr_anon_transparent_hugepages" $F
#	F=$(grep -l "# /sys/devices/system/node/node3/vmstat" "${TEMP}"/${FIL}/xx* | tr -d ":#" | awk '{print $1}')
#		test -z "$F" ||\
#			grep "nr_anon_transparent_hugepages" $F
#	F=$(grep -l "# /sys/devices/system/node/node4/vmstat" "${TEMP}"/${FIL}/xx* | tr -d ":#" | awk '{print $1}')
#		test -z "$F" ||\
#			grep "nr_anon_transparent_hugepages" $F
#	F=$(grep -l "# /sys/devices/system/node/node5/vmstat" "${TEMP}"/${FIL}/xx* | tr -d ":#" | awk '{print $1}')
#		test -z "$F" ||\
#			grep "nr_anon_transparent_hugepages" $F
#	F=$(grep -l "# /sys/devices/system/node/node6/vmstat" "${TEMP}"/${FIL}/xx* | tr -d ":#" | awk '{print $1}')
#		test -z "$F" ||\
#			grep "nr_anon_transparent_hugepages" $F
#	F=$(grep -l "# /sys/devices/system/node/node7/vmstat" "${TEMP}"/${FIL}/xx* | tr -d ":#" | awk '{print $1}')
#		test -z "$F" ||\
#			grep "nr_anon_transparent_hugepages" $F
#	F=$(grep -l "# /sys/devices/system/node/node8/vmstat" "${TEMP}"/${FIL}/xx* | tr -d ":#" | awk '{print $1}')
#		test -z "$F" ||\
#			grep "nr_anon_transparent_hugepages" $F
#
#	F=$(grep -l "# /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" "${TEMP}"/${FIL}/xx* | tr -d ":#" | awk '{print $1}')
#		test -z "$F" ||\
#			cat $F
	echo
	echo "SAP recommended cpuidle: acpi_idle"
	grep -A1 "# /sys/devices/system/cpu/cpuidle/current_driver" sysfs.txt
	echo

	FIL="fs-diskio.txt"
	test -f ${FIL} &&\
		split_file ${FIL}
	
	echo "SAP recommeded filesystem type: XFS"
	F=$(grep -l "# /etc/fstab" "${TEMP}"/${FIL}/xx* | tr -d ":#" | awk '{print $1}')
	# shellcheck disable=SC2086
	grep -v "#==" $F | tr -s " "
	# TODO XFS create options

	# TODO calculate 2x RAM, max.50GB
	echo "SAP recommended swap space: 2x RAM, max.50GB"
	# TODO depends on name "swap"
	grep "Disk.*swap" ${FIL}

	# grep "pagecache.limit.set"
	# TODO search Fusion IO drives 
	# TODO boot msg shows max.4GB, use other info here	
	FIL="boot.txt"
	grep "^\[.*System RAM: " ${FIL} | awk '{print $11,$12,$13}'

	echo

	FIL="chkconfig.txt"
	if test -r "${FIL}"; then
		echo "SAP recommeded service ON:"
		for s in $HANA_SAP_SVC_YES; do
			echo -n "$s: "
			p=$(test -r ${FIL} && grep "${s}:" ${FIL} | cut -d: -f2)
			# TODO
			# p=$(test -r ${FIL} && grep "${s}.*0:" ${FIL} | cut -d: -f2-)
			test -z "$p" && p="NA"
			echo "${p}"
		done
		echo
		echo "SAP recommeded service OFF:"
		for s in $HANA_SAP_SVC_OFF; do
			echo -n "$s: "
			p=$(test -r ${FIL} && grep "${s}:" ${FIL} | cut -d: -f2)
			# TODO
			# p=$(test -r ${FIL} && grep "${s}.*0:" ${FIL} | cut -d: -f2-)
			test -z "$p" && p="NA"
			echo "${p}"
		done
	fi
	FIL="systemd.txt"
	if test -r "${FIL}"; then
		# TODO SLES12
		echo
	fi
	echo
	FIL="memory.txt"
	split_file ${FIL}
	f=$(grep -l "^# /usr/bin/pmap [1-9]" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
        test -z "$f" ||\
		awk ' $9~/HDB_/ {print $9}' $f | sort -u | tail -2
	echo
	FIL=pam.txt
	split_file ${FIL}
	show_pam_sap
	echo
}


function chk_hpehana(){

	echo_msgsep "$FUNCNAME"

	test -z "${HPE_HANA_PTRN}" &&\
	HPE_HANA_PTRN="
after.local
"
	test -z "${HPE_HANA_NIC}" &&\
	HPE_HANA_NIC="
"p3p[12]"
"p6p[12]"
"p[0-9]...[0-9]p[12]"
"bond[0-7]"
"em[0-9][0-9]"
hana.prod
hana.prd
hana.nfs
hana.int
hana.qor
hana.rep
"
# TODO check and complete HPE_HANA_PKG
	test -z "${HPE_HANA_PKG}" &&\
	HPE_HANA_PKG="
HPE-HANA-Bundle-Release
csDrift
csurUpdate
firmware-cna-emulex
firmware-fc-qlogic
firmware-hca-mellanox-vpi-eth-ib
firmware-nic-broadcom
firmware-system-u17
firmware-system-p89
hp-firmware-fc-qlogic
hpsmh
hp-ams
hp-smx
mlnx-ofa_kernel
mlnx-ofa_kernel-kmp-default
rear
serviceguard
serviceguard-extension-for-sap
"
	FIL="rpm.txt"
	echo "RPMs for HPE HANA appliance:"
	# TODO better pattern for package versions
	for f in $HPE_HANA_PKG; do
		echo -n "$f : "
		grep -v -- "-32bit" ${FIL} |\
		grep -e "^${f}-[0-9]" -e "^${f}-[A-Z].[0-9]" || echo "NO"
        done | awk '{print $1,":",$3}' | sort
	echo

	FIL=boot.txt
	echo "/etc/init.d/after.local:"
	split_file ${FIL}
	f=$(grep -l "^# /etc/init.d/after.local" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		grep -v "#==..Configuration.File" $f
	rm -rf "${TEMP:?}"/${FIL}
	echo

	FIL=tuned.txt
	split_file ${FIL}
	f=$(grep -l "^# /usr/lib/tuned/sap-hpe-hana/tuned.conf" "${TEMP}"/${FIL}/xx*) 
	# shellcheck disable=SC2086
	test -z "$f" ||\
		cat $f
	rm -rf "${TEMP:?}"/${FIL}
	echo
	
	echo "NICs:"
	echo

	FIL=network.txt
	split_file ${FIL}
	f=$(grep -l "# /usr/sbin/wicked ifstatus --verbose all" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" || (
		echo -n "number of slave devices: "
		grep -c "\ enslaved$" $f
		echo -n "number of bond devices: "
		grep -c "type:.....bond, mode " $f
		echo -n "number of vlans: "
		grep -c "type:.....vlan bond.*, " $f
		echo	
		grep -v "^#" $f |\
		# TODO better exclusion
		grep -v "uuid:" | grep -v "route:.*ipv" | grep -v "leases:" |\
		grep -v "addr:.*ipv6"
	)
	rm -rf "${TEMP:?}"/${FIL}
	echo

	FIL=hardware.txt
	# /usr/sbin/dmidecode
	# /sbin/lspci -nn
	# /sbin/lspci -vvv
	# /usr/bin/lsusb -vv
	# /bin/ls -al /dev | egrep "fd.*|dvd.*|cdr.*"
	split_file ${FIL}
	f=$(grep -l "# /usr/bin/dmidecode" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		for p in $HPE_HANA_NIC; do
			echo "p: $p   f: $f"
			# TODO meaningfull stanza limits
			grep -A40 "SysFS.ID:./class/net/$p" $f |\
			grep -e "Link.detected:" -e "Driver:"
		done
	f=$(grep -l "# /usr/sbin/hwinfo -braille" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		for p in $HPE_HANA_NIC; do
			echo "$p"
			# TODO meaningfull stanza limits
			# TODO NIC drivers as HPE_... variable
			# TODO "net.interface..name." does not exist for all drivers
			grep -A40 "net.interface..name.=.$p" $f |\
			grep -e "ethtool.driver:" -e "mlx._.*:" -e "be2net:" -e "ixgbe:" 
		done
	rm -rf "${TEMP:?}"/${FIL}
	echo

	# TODO make it work
	# /usr/bin/systool -vc net
	# /usr/bin/systool -vb pci

	FIL=sysfs.txt
	split_file ${FIL}
	f=$(grep -l "# /usr/bin/systool -vc net" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		for p in $HPE_HANA_NIC; do
			# TODO meaningfull stanza limits
			grep -v "00 00 00 00 00 00 00 00" $f |\
			grep -A90 "Class.Device.=..$p" |\
			grep -e "Class.Device.=" -e "mtu.*=" -e "speed.*=" -e "tx_.*=" -e "rx_.*=" -e "duplex.*=" -e "flags.*=" -e "mlx._port.*=" -e "numa_node.*="
		done
	echo
	f=$(grep -l "# /usr/bin/systool -vb pci" "${TEMP}"/${FIL}/xx*)
		# shellcheck disable=SC2086
		for p in $HPE_HANA_NIC; do
			# TODO meaningfull stanza limits
			grep -v "00 00 00 00 00 00 00 00" $f |\
			grep -A90 "Class.Device.=..$p" |\
			grep -e "Class.Device.=" -e "mtu.*=" -e "speed.*=" -e "tx_.*=" -e "rx_.*=" -e "duplex.*=" -e "flags.*=" -e "mlx._port.*=" -e "numa_node.*="
		done
	# TODO what about messages-YYYYMMDD.txt?
	rm -rf "${TEMP:?}"/${FIL}

	FIL=messages.txt
	split_file ${FIL}
	f=$(grep -l "# /var/log/messages" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		for p in $HPE_HANA_PTRN; do
			grep $p $f
		done
	rm -rf "${TEMP:?}"/${FIL}
	echo
}


function chk_oracle(){

	echo_msgsep "$FUNCNAME"

	# TODO put this in config file, use it in function
	# TODO see /srv/pinne/work/*/sap/technical/from_*/20130304/ps_*.txt
# orawe3    4637  0.0  2.4 10495496 3260024 ?    Ss   Mar03   3:21 oracleWE3 (LOCAL=NO)
# orawei    5038  0.0  0.0 10489296 19884 ?      Ss   Feb24   0:21 ora_q001_WEI
# orawe3   19399  0.0  0.0 10491972 29064 ?      Ss   Mar03   1:20 ora_pmon_WE3
# orawe3   19402  0.0  0.0 10489520 17080 ?      Ss   Mar03   0:58 ora_psp0_WE3
# orawe3   20152  0.0  0.0 10489520 16560 ?      Ss   Mar03   0:27 ora_diag_WE3
# orawe3   20175  0.0  0.0 10490032 41084 ?      Ss   Mar03   0:14 ora_dbrm_WE3
# orawe3   20191  0.2  0.0 10494128 44176 ?      Ss   Mar03  14:36 ora_dia0_WE3
# orawe3   20229  0.0  0.0 10489520 84100 ?      Ss   Mar03   0:13 ora_mman_WE3
# orawe3   20271  0.0  2.7 10496556 3600380 ?    Ss   Mar03   3:38 ora_dbw0_WE3
# orawet   20296  0.3  1.4 4977156 1958052 ?     Ss   Feb27  35:23 oracleWET (LOCAL=NO)
# orawe3   20310  0.0  2.7 10495532 3635364 ?    Ss   Mar03   3:36 ora_dbw1_WE3
# orawe3   20331  0.0  2.8 10496044 3794112 ?    Ss   Mar03   3:36 ora_dbw2_WE3
# orawe3   20357  0.0  2.7 10495532 3617344 ?    Ss   Mar03   3:34 ora_dbw3_WE3
# orawe3   20381  0.0  2.7 10496044 3653224 ?    Ss   Mar03   3:35 ora_dbw4_WE3
# orawe3   20402  0.6  0.0 10512176 39104 ?      Ss   Mar03  35:23 ora_lgwr_WE3
# orawe3   20422  0.0  0.1 10490536 210024 ?     Ss   Mar03   2:25 ora_ckpt_WE3
# orawe3   20439  0.0  0.7 10491064 925884 ?     Ss   Mar03   0:37 ora_smon_WE3
# orawe3   20459  0.0  0.0 10490032 35476 ?      Ss   Mar03   0:07 ora_reco_WE3
# orawe3   20488  0.0  0.2 10495560 352076 ?     Ss   Mar03   0:34 ora_mmon_WE3
# orawe3   20523  0.1  0.1 10489520 150708 ?     Ss   Mar03   7:05 ora_mmnl_WE3
# orawe2   25496  0.0  0.0  83832 12588 ?        Ssl  Feb24   0:25 /oracle/WE2/112_64/bin/tnslsnr listener_WE2 -inherit
# root     25625  4.7  0.0   6240  1176 ?        S    16:40   0:00 /oracle/WE3/libelle/trd/bin/trd
# orawe3   25926  0.0  0.0 10489512 20492 ?      Ss   Mar03   0:17 ora_smco_WE3
# orawe3   26972  0.0  0.0 10491052 33200 ?      Ss   Mar03   0:07 ora_q000_WE3
# orawe2   27280  0.0  0.0 6885040 17004 ?       Ss   Feb24   2:46 ora_psp0_WE2
# orawe2   31816  0.0  0.0 6885040 20592 ?       Ss   Feb24   0:25 ora_qmnc_WE2
	# 
	# shellcheck disable=SC2034
        ORA_PRCS="
ora_pmon_
ora_vktm_
ora_gen._
ora_diag_
ora_dbrm_
ora_dia._
ora_dbw._
ora_lgwr_
ora_ckpt_
ora_smon_
ora_reco_
ora_mmon_
ora_mmnl_
ora_smco_
ora_psp._
ora_q.*_
ora_qmnc_
oracle.*
/.*/bin/tnslsnr
"

	echo "standalone Oracle installation"
	echo
	echo "sysconfig_oracle :"
        FIL="sysconfig.txt"
        split_file ${FIL}
	# TODO fix test
        F=$(grep -l "# /etc/sysconfig/oracle" "${TEMP}"/${FIL}/xx* |\
		 tr -d ":#" | awk '{print $1}')
	# shellcheck disable=SC2086
	test -z $F ||\
		grep -v "^#" $F  | awk -F: '{print $2}' | tr -s "\n "
        echo

	grep -e "^oracle" -e "^@dba" crash.txt
	echo
	echo "tmpfs:"
	echo "   Size  Used  Avail  Use%  Mount"
	echo "   Inodes  IUsed  IFree  IUse%  Mount"
	grep "^tmpfs" basic-health-check.txt | tr -s " "
	echo		
	grep "^vm.nr_hugepages.=" env.txt	
	echo
	grep -A1 "^# /sys/kernel/mm/transparent_hugepage/enabled" memory.txt |\
		tr -d "#" 
	echo
	# TODO limits.conf: soft memlock, hard memlock
	# TODO http://www.oracle-base.com/articles/linux/configuring-huge-pages-for-oracle-on-linux-64.php
	# TODO check ???, see sum_base_config, whbsaprecheck
	# TODO check async-IO, direct-IO
	# TODO hugetblfs from mount
	# TODO check Oracle alert file for several "Large.Pages.*=" 
	# TODO oracleasmlib oracleasm-support still needed?	

	FIL="rpm.txt"
        test -z "${ORA_SFT_PTRN}" &&\
        ORA_SFT_PTRN="
orarun
oracleasmlib
oracleasm-support
oracleasm-kmp-default
oracle-instantclient11.2-basic
oracle-instantclient11.2-basiclite
oracle-instantclient11.2-jdbc
oracle-instantclient11.2-sqlplus
oracle-instantclient11.2-devel
oracle-instantclient11.2-odbc
oracle-instantclient11.2-tools
cups-client
cups-drivers
cups-libs
cups-libs-32bit
cups
cups-autoconfig
xorg-x11-libs
xorg-x11
nfs-client
glibc
glibc-32bit
glibc-locale
glibc-locale-32bit
glibc-devel
kernel-default
linux-kernel-headers
binutils
gcc
gcc-32bit
gcc43
gcc-locale
gcc-locale
libgcc43
libgcc43-32bit
libaio
libaio-devel
sysstat
supportutils
suse-sam
multipath-tools
mdadm
device-mapper
device-mapper-32bit
libext2fs2
ClusterTools2
"
	echo "RPM needed :"
	# shellcheck disable=SC2086
        for p in $ORA_SFT_PTRN; do
                echo ${p}
        done
	echo
	echo "RPM found :"
	# shellcheck disable=SC2086
        for p in $ORA_SFT_PTRN; do
                awk '$1=="'${p}'" {print $0}' ${FIL}
        done
        echo
# TODO chrony
	test -z "${ORA_SVC_YES}" &&\
	ORA_SVC_YES="
boot.clock
boot.dmraid
boot.md
boot.devicemapper
boot.lvm
boot.multipath
boot.sysstat
boot.sysctl
multipathd
mdadmd
ntp
oracle
"
	test -z "${ORA_SVC_OFF}" &&\
	ORA_SVC_OFF="
boot.sapconf
alsasound
ipmi
openais
powerd
slpd
smartd
sapinit
"
	FIL="chkconfig.txt"
	if test -r "${FIL}"; then
		echo "service needed :"
		for s in $ORA_SVC_YES; do
			echo "$s"
		done
		echo
		echo "service found :"
		for s in $ORA_SVC_YES; do
			grep "${s}:..on" ${FIL}
			grep "${s}:..off" ${FIL}
			grep "^${s}.*0:.*3:on" ${FIL} | grep -v "boot\."
			grep "^${s}.*0.*3:off" ${FIL} | grep -v "boot\."
		done
		echo
		echo "service should be off :"
		for s in $ORA_SVC_OFF; do
			grep "${s}:..off" ${FIL}
			grep "${s}:..on" ${FIL}
			grep "^${s}.*0.*3:off" ${FIL} | grep -v "boot\."
			grep "^${s}.*0.*3:on" ${FIL} | grep -v "boot\."
		done
	fi
	FIL="systemd.txt"
	if test -r "${FIL}"; then
		# TODO SLES12
		echo
	fi
        echo
	# TODO rpm.txt /usr/local/bin: coraenv, dbhome, oraenv
	echo "proc :"
	FIL=memory.txt
	split_file ${FIL}
	f=$(grep -l "^# /usr/bin/pmap [1-9]" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		awk '$8~/dboraslib/ || $8~/libaio/ || $8~/libclntsh.so/ || $8~/tnslsnr/ || $8~/libodmd11/ || $8~/oracle$/ {print $8}' $f | sort -u
	echo
	# TODO oracle user, pfile
	# TODO memory.txt: tnslsnr libclntsh oracle$ libodmd11
	# TODO env.txt: ORA_
	FIL=env.txt
	split_file ${FIL}
	f=$(grep -l "^# /usr/bin/env$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		awk '$1~/ORA/ {print}' $f | sort -u
	echo
}


function chk_kernel(){
	
	echo_msgsep "$FUNCNAME"

	FIL="boot.txt"
	grep -A1 "^# /bin/uname -a" ${FIL}
	grep -A1 "^# /proc/cmdline" ${FIL}
	grep "^\[.*Booting paravirtualized kernel on bare hardware" ${FIL}
	echo -n "kernel.tainted : "
	grep "^kernel.tainted.=" env.txt

	FIL="basic-health-check.txt"
	test -r ${FIL} &&\
	grep -e "TAINT:.\(.\)" -e "^Kernel.Status.--" ${FIL}
	
	echo
}


function chk_software(){

	echo_msgsep "$FUNCNAME"

	FIL="basic-environment.txt"
	grep -A7 "# /etc/os-release" ${FIL}
	echo
	grep -A1 "# /etc/lsb-release" ${FIL}
	echo	

	FIL="rpm.txt"
	grep "(none)" ${FIL} | grep -v "gpg-pubkey"
	echo	

	# TODO sam is gone
	FIL="sam.txt"
	test -z "${SOFTW_PTRN}" &&\
	SOFTW_PTRN="
System.architecture:
Baseproduct:
not.our.package:
"
	for p in $SOFTW_PTRN; do
		grep "${p}" ${FIL}
	done
	echo
}


function chk_updates(){

	echo_msgsep "$FUNCNAME"
	
	FIL="updates.txt"
	# TODO sles-for-sap
	# TODO fix doubled entries artefact "SLES12-*"
	# TODO fix SLE-12 patterns, add SLE-15
	test -z "${UPDT_PTRN}" &&\
	UPDT_PTRN="
"SLES12-"
"SLES12-SP"
"SLE12-HAE-SP"
"SLE12-SP1-SAP-"
"SLE12-SP2-SAP-"
"SLE12-SP3-SAP-"
"SLE12-SP4-SAP-"
"SLE12-SP5-SAP-"
"
	for p in ${UPDT_PTRN}; do
		grep "|.*|.${p}.*|.*|" ${FIL}
	done | sort -u
	echo
	grep "patches.needed" ${FIL}
	echo
}


function chk_multipath(){

	echo_msgsep "$FUNCNAME"

	FIL="mpio.txt"
	split_file ${FIL}
	F=$(grep -l "# /etc/multipath.conf" "${TEMP}"/${FIL}/xx* | tr -d ":#" | awk '{print $1}')
	echo "# /etc/multipath.conf:"
	# shellcheck disable=SC2086
	test -z "$F" || grep -v "^#" $F | tr -s "\n"

	for p in "^#.rpm.-V" "^#.Verification.Status:"; do
		grep "${p}" ${FIL}
	done
	grep "^#.RPM.Not.Installed:" ${FIL}
	echo

	for p in "^boot.device-mapper.*o" "^boot.multipath.*o" "^multipathd.*0:.*3:on"; do
		grep "${p}" ${FIL}
	done
	echo
	
	grep -B1 "^size=.*features=.*hwhandler=" ${FIL} |\
		sed s/^size/"	size"/g | grep -v "^--$"
	echo
	# TODO by-id
	# TODO dmsetup
	# TODO show complete multipath.conf
	# TODO multipath.conf: compare failback, rr_min_io, rr_min_io_rq, no_path_retry, polling_interval, dev_loss_tmo, user_friendly_names, async???
	# TODO check for un-matched userfriendly names (mpath[a-z])
	for p in "loading.*prioritizer" "path.checker.=" "prio.="; do
		grep "${p}" ${FIL}
	done | awk '{print $6,$7,$8}' | sort -u
	echo
}
		

function chk_filesys(){

	echo_msgsep "$FUNCNAME"

	FIL="fs-diskio.txt"
	split_file ${FIL}
	F=$(grep -l "# /etc/fstab" "${TEMP}"/${FIL}/xx* | tr -d ":#" | awk '{print $1}')
	# shellcheck disable=SC2086
	grep -v "#==" $F | tr -s " "
	F=$(grep -l "# /bin/mount" "${TEMP}"/${FIL}/xx* | tr -d ":#" | awk '{print $1}')
	# shellcheck disable=SC2086
	grep -v "#==" $F | tr -s " "
	# TODO define meaningful limits for await and svctm	
	# TODO make limits for await and svctm a config parameter
	# TODO autofs	
	F=$(grep "# /usr/bin/iostat" "${TEMP}"/${FIL}/xx* | tr -d ":#" | awk '{print $1}')
	echo "$F" | awk ' $1=="Device:" {print $0}
		$10 > 100.0 {print $0}
		$11 > 20.0 {print $0}' | sort -u
	echo

	FIL="fs-btrfs.txt"
	split_file ${FIL}
	( grep -A4 "# /sbin/btrfs filesystem show" "${TEMP}"/${FIL}/xx*
	grep -A5 "# /sbin/btrfs scrub status /$" "${TEMP}"/${FIL}/xx*
	grep -A5 "# /sbin/btrfs filesystem df /$" "${TEMP}"/${FIL}/xx*
	grep -A5 "# /sbin/btrfs filesystem df /var/log$" "${TEMP}"/${FIL}/xx*
	grep -A5 "# /sbin/btrfs filesystem df /tmp$" "${TEMP}"/${FIL}/xx*
	grep -A5 "# /sbin/btrfs filesystem df /.snapshots$" "${TEMP}"/${FIL}/xx* ) | tr "-" ":" | awk -F: '{print $3,$4}'
}


function chk_storage(){
	
	echo_msgsep "$FUNCNAME"

	FIL="rpm.txt"
	# TODO see whbsaprecheck
	test -z "${STOR_PKG}" &&\
	STOR_PKG="
dmraid
lvm2
libext2fs2
mdadm
multipath-tools
nfs-client
open-iscsi
ocfs2-tools
ocfs2-kmp-default
xfsprogs
"
	echo "software:"
	for p in $STOR_PKG; do
		awk '$1=="'"$p"'" {print $1,$NF} ' ${FIL}
	done
	echo
	# TODO check if local disk is in multipath
	# TODO nfs
	# TODO iSCSI, LIO for SBD, timings?
	test -z "${STOR_MOD}" &&\
	STOR_MOD="
btrfs
ext2
ext3
ext4
ocfs2
xfs
"	
	echo "disk:"
	FIL="fs-diskio.txt"
	#grep -B1 "^Disk./dev/" ${FIL}
	grep -A1 "^Model:" ${FIL} | grep -v "^--$"
	echo
	show_sys_scsi
	echo

	# TODO FIL="fs-softraid.txt"
	# TODO better elevator guessing 
	# TODO e.g. udev.txt: # /usr/lib/udev/rules.d/60-ssd-scheduler.rules -> deadline
	FIL="boot.txt"
	echo "/boot/grub/menu.lst:"
	grep "io.scheduler.*registered.(default)" ${FIL}
	grep "kernel.*elevator" ${FIL}
	echo
	FIL="sysconfig.txt"
	echo "/etc/sysconfig/bootloader:"
	grep "DEFAULT_APPEND.*elevator" ${FIL}
	echo
	split_file ${FIL}
	echo "/etc/sysconfig/mdadm:"
	f=$(grep -l "# /etc/sysconfig/mdadm" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
        test -z "$f" || grep -v "^#" $f | tr -s "\n"
        echo
	
	FIL="fs-softraid.txt"
	split_file ${FIL}
	echo "/etc/sysconfig/mdadm:"
	f=$(grep -l "# /etc/mdadm.conf" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086,SC2002
        test -z "$f" || cat $f | tr -s "\n"
        echo
	echo "/proc/mdstat:"
	f=$(grep -l "# /proc/mdstat" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086,SC2002
        test -z "$f" || cat $f | tr -s "\n"
        echo

	FIL="lvm.txt"
	split_file ${FIL}
	echo "/sbin/vgs:"
        f=$(grep -l "# /sbin/vgs" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" || grep -vh "^#" $f | tr -s "\n"
	echo
	echo "/etc/sysconfig/lvm:"
        f=$(grep -l "# /etc/sysconfig/lvm" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" || grep -vh "^#" $f | tr -s "\n"
	echo
	echo "/etc/lvm/lvm.conf:"
	f=$(grep -l "# /etc/lvm/lvm.conf" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" || grep -vh "^#" $f | grep -vh "^    #" | tr -s "\n"
	echo
}


function chk_overview(){
	
	echo_msgsep "$FUNCNAME"

	#FIL="supportconfig.txt"
	#grep "Script.Version:" ${FIL}
	FIL="network.txt"
	echo -n "Hostname: "
	grep --binary-file=text -A1 "#./etc/HOSTNAME" ${FIL} | tail -1
	# TODO replace basic-health-report.txt by basic-health-check.txt
	# TODO no basic-health-report.txt in SLES12
	# TODO D state, defunc (zombie), ...
	FIL="basic-health-report.txt"
	echo "Supportconfig: $DIR"
	test -r ${FIL} &&\
	for p in "^Date.Checked:" "^Status:" "\[...Red...\]" "\[.Yellow..\]"; do
		grep "${p}" ${FIL}
	done
	echo
}


function chk_sysconf(){

	echo_msgsep "$FUNCNAME"

	SYSC_PTRN="
fs.file-max
fs.aio-max-nr
kernel.panic_on_oops
kernel.panic
kernel.sem
kernel.shmall
kernel.shmmax
kernel.shmmni
kernel.threads-max
net.ipv4.ip_local_port_range
net.ipv4.tcp_max_syn_backlog
net.ipv4.tcp_slow_start_after_idle
net.ipv4.tcp_window_scaling
net.core.rmem_default
net.core.rmem_max
net.core.wmem_default
net.core.wmem_max
net.core.somaxconn
vm.dirty_*
vm.zone_reclaim_mode
vm.numa_zonelist_order
vm.nr_pdflush_threads
vm.min_free_kbytes
vm.nr_hugepages
vm.swappiness
vm.overcommit_memory
vm.max_map_count
vm.hugetlb_shm_group
vm.pagecache_limit_*
vm.vfs_cache_pressure
"
	# TODO get really active settings
#	FIL="proc.txt"
#	PROC_PTRN=$(echo $SYSC_PTRN | tr "." "/" )
	FIL="env.txt"
	PROC_PTRN=$SYSC_PTRN
	echo "proc : "
	for m in ${PROC_PTRN}; do
#		grep -A1 "${m}" ${FIL}
		grep "^${m}" ${FIL}
	done | tr -s " " | tr "	" " " | sort -u
	echo
	FIL="etc.txt"
	echo "sysctl.conf : "
	for m in ${SYSC_PTRN}; do	
		grep "^${m}" ${FIL}
	done | sort -u
	echo
	# TODO security/limits.conf
	# TODO tmpfs
	FIL="env.txt"
	split_file ${FIL}
	F=$(grep -h "# ulimit -a" "${TEMP}"/${FIL}/xx* | tr -d ":#")
        echo "$F" | grep -v "#==" | tr -s " "

	FIL="sysconfig.txt"
	split_file ${FIL}
	F=$(grep -l "# /etc/sysconfig/ulimit" "${TEMP}"/${FIL}/xx* | tr -d ":#" |\
		 awk '{print $1}')
	# TODO make it work:
	# test -r $F && grep -v "^#" $F | tr -s "\n "
	echo

	# TODO /sys/kernel/mm/transparent_hugepage/enabled
	# TODO /sys/kernel/mm/transparent_hugepage/defrag

	# TODO how to find scheduler from supportdonfig? plugin needed?
	FIL="boot.txt"
	echo -n "io_scheduler: "; grep "io scheduler.*registered..default" ${FIL}
	echo
}


function chk_sysload(){

	echo_msgsep "$FUNCNAME"

	# Thanks ralph.roth@suse.com
	echo "basic-health-check:"
	FIL="basic-health-check.txt"
	# TODO make dfull work on SLE12 and 15
	dfull="X"
	dfull=$( test -r ${FIL} && grep "%.>=.90%" ${FIL} )
	test "${dfull}" = "X" &&\
	(	echo "Used Disk Space:"
		echo "$dfull"
	)
	echo
	
	# TODO hardare.txt:
	FIL="hardware.txt"
	grep -v "^#" $FIL | head -17 | tail -15

	OWD=$PWD
	cd sar || exit
	# TODO sa?? now is sar20190309.xz.
	# sar20190309 plain ASCII is only about the rpm itself.
	# TODO print date of sar file to each output line
	#	either by adding date from sar-fiel header,
	#	or by grep-ing the alreday known maxima out of the files
	# TODO reduce number of columns

	# to support filenames like sa23 or sa20191104
	# if we use extended globbing (add 'shopt -s extglob' right under the
	# shebang line), we can use a more elegant way to collect the files
	# for i in sa+([[:digit:]]); do
	for i in sa?? sa????????; do
		test -f "$i" && sarFiles="$sarFiles $i"
	done
	
	echo "max. load-avg15:"
	echo "hh:mm:ss xM   runq-sz  plist-sz   ldavg-1   ldavg-5  ldavg-15"
	for f in $sarFiles; do
		sar -q -f "$f"
	done | grep "^[0-9][0-9]:" | sort -n -k7 | tail

	echo "max. user CPU utilisation:"
	echo "hh:mm:ss xM     CPU     %user     %nice   %system   %iowait    %steal     %idle"
	for f in $sarFiles; do
		sar -f "$f"
	done | grep "^[0-9][0-9]:" | grep -v "%" | sort -n -k4 | tail

#	echo "max. system CPU utilisation:"
#	echo "hh:mm:ss xM     CPU     %user     %nice   %system   %iowait    %steal     %idle"
#	for f in sa??; do
#		sar -f $f 
#	done | grep "^[0-9][0-9]:" | grep -v "%" | sort -n -k6 | tail

	echo "max. iowait CPU utilisation:"
	echo "hh:mm:ss xM     CPU     %user     %nice   %system   %iowait    %steal     %idle"
	for f in $sarFiles; do
		sar -f "$f"
	done | grep "^[0-9][0-9]:" | grep -v "%" | sort -n -k7 | tail

#	echo "max. interrupts:"
#	echo "hh:mm:ss xM       INTR    intr/s"
#	for f in sa??; do
#		sar -I SUM -f $f
#	done | grep "^[0-9][0-9]:" | sort -n -k4 | tail

#	echo "max. disk write tps:"
#	echo "hh:mm:ss xM       tps      rtps      wtps   bread/s   bwrtn/s"
#	for f in sa??; do
#		sar -f $f 
#	done | grep "^[0-9][0-9]:" | sort -n -k5 | tail

	echo "max. disk write blocks:"
	echo "hh:mm:ss xM       tps      rtps      wtps   bread/s   bwrtn/s"
	for f in $sarFiles; do
		sar -f "$f"
	done | grep "^[0-9][0-9]:" | sort -n -k7 | tail

#	echo "max. disk read tps:"
#	echo "hh:mm:ss xM       tps      rtps      wtps   bread/s   bwrtn/s"
#	for f in sa??; do
#		sar -f $f 
#	done | grep "^[0-9][0-9]:" | sort -n -k4 | tail

	echo "max. disk read blocks:"
	echo "hh:mm:ss xM       tps      rtps      wtps   bread/s   bwrtn/s"
	for f in $sarFiles; do
		sar -f "$f"
	done | grep "^[0-9][0-9]:" | sort -n -k6 | tail

	echo "max. swap used:"
	echo "hh:mm:ss xM kbswpfree kbswpused  %swpused  kbswpcad   %swpcad"
	for f in $sarFiles; do
		sar -f "$f"
	done | grep "^[0-9][0-9]:" | sort -n -k4 | tail

	echo "max. pageout:"
	echo "hh:mm:ss xM pgpgin/s pgpgout/s   fault/s  majflt/s  pgfree/s pgscank/s pgscand/s pgsteal/s    %vmeff "
	for f in $sarFiles; do
		sar -B -f "$f"
	done | grep "^[0-9][0-9]:" | sort -n -k4 | tail

#	# find lowest vmeff% that is not 0.00. 0.00 means "nothing done".
#	echo "min. vmeff:"
#	echo "hh:mm:ss xM pgpgin/s pgpgout/s   fault/s  majflt/s  pgfree/s pgscank/s pgscand/s pgsteal/s    %vmeff "
#	for f in sa??; do
#		sar -B -f $f 
#	done | grep "^[0-9][0-9]:" | grep -v "%" |\
#	 	awk '$11 != "0.00" {print}' | sort -n -k11 | head

	# TODO sum up buffer+cache, estimated use memory 
	echo "max. memory commit:"
	echo "hh:mm:ss xM kbmemfree kbmemused  %memused kbbuffers  kbcached  kbcommit   %commit"
	for f in $sarFiles; do
		sar -r -f "$f"
	done | grep "^[0-9][0-9]:" | sort -n -k8 | tail

	echo "max. memory cache:"
	echo "hh:mm:ss xM kbmemfree kbmemused  %memused kbbuffers  kbcached  kbcommit   %commit"
	for f in $sarFiles; do
		sar -r -f "$f"
	done | grep "^[0-9][0-9]:" | sort -n -k7 | tail

#	echo "max. network write tps:"
#	echo "hh:mm:ss xM     IFACE   rxpck/s   txpck/s    rxkB/s    txkB/s   rxcmp/s   txcmp/s  rxmcst/s"
#	for f in sa??; do
#		sar -n DEV -f $f 
#	done | grep "^[0-9][0-9]:" | grep bond | sort -n -k5 | tail

	echo "max. network write kB:"
	echo "hh:mm:ss xM     IFACE   rxpck/s   txpck/s    rxkB/s    txkB/s   rxcmp/s   txcmp/s  rxmcst/s"
	for f in $sarFiles; do
		sar -n DEV -f "$f"
	done | grep "^[0-9][0-9]:" | grep bond | sort -n -k7 | tail

#	echo "max. network read tps :"
#	echo "hh:mm:ss xM     IFACE   rxpck/s   txpck/s    rxkB/s    txkB/s   rxcmp/s   txcmp/s  rxmcst/s"
#	for f in sa??; do
#		sar -n DEV -f $f 
#	done | grep "^[0-9][0-9]:" | grep bond | sort -n -k4 | tail

	echo "max. network read kB:"
	echo "hh:mm:ss xM     IFACE   rxpck/s   txpck/s    rxkB/s    txkB/s   rxcmp/s   txcmp/s  rxmcst/s"
	for f in $sarFiles; do
		sar -n DEV -f "$f"
	done | grep "^[0-9][0-9]:" | grep bond | sort -n -k6 | tail

	# TODO network errors?
	# TODO NFS client and server reports?
	# TODO grep -A6 "numastat" sysctl.txt 

	cd "$OWD" || exit
	echo
}


function chk_errors(){
	
	echo_msgsep "$FUNCNAME"

	# TODO see cs_show_error_patterns
	test -z "${ERRCFG}" &&\
		ERRCFG="/etc/ClusterTools2/cs_show_error_patterns"
	test -s "$ERRCFG" && source "$ERRCFG"
 
	# TODO taint.flag ?
	echo -n "kernel.tainted : "
	grep "^kernel.tainted.=" env.txt

	# TODO add plugin-messages.txt, man supportconfig for plugin
	# TODO find start and end time for each log files

	FIL="messages.txt"
	# TODO fix messages.txt starts <NUL>
	FRST=$(grep -A1 "# /var/log/messages - Last.*Lines" ${FIL} |\
		tail -1 | awk '{print $1,$2,$3}')
	echo "${FIL} starts $FRST"
	
	LOG="boot.txt messages.txt proc.txt ha.txt"
	# shellcheck disable=SC2086
	for e in ${ERROR_PATTERN} ${CLUERR_PATTERN}; do
		echo -n "${e} : "
		for f in ${LOG}; do
			cat "$f"
		done | grep -ic ${e}
	done
	echo

	# TODO dropped packets from proc.txt
	echo "dropped network packets:"
	FIL="proc.txt"
	split_file ${FIL}
	f=$(grep -l "^# /proc/net/dev$" "${TEMP}"/${FIL}/xx*)
	# shellcheck disable=SC2086
	test -z "$f" ||\
		cat $f

NET_ERR="
ICMP.message.failed
packets.pruned.from.receive.queue
delayed.acks
listen.queue.of.a.socket.overflowed
"
	FIL="network.txt"
	for p in $NET_ERR; do
		grep "$p" ${FIL}
	done
	echo
}


# main()

case $1 in
	--help|-h)
		BEX=$(basename "$EXE")
		echo "usage: $BEX OPTION"
		echo "usage: $BEX --generic <DIR>"
		echo "usage: $BEX --function <DIR> <FUNC>"
		echo
		echo "OPTION:"
		echo " --help		show help."
		echo " --listfunc	show available functions."
		echo " --version	show version."
		echo " --writecfg	show some internal variables."
		echo " --generic	show generic info."
 		echo " --function	show info from selected function."
		exit
	;;
	--version|-v)
		echo -n "$(basename "$EXE") "
		head -11 "$EXE" | grep "^# Version: "
		exit
	;;
	--listfunc|-l)
		awk '$1=="function" && $2~/^chk_/ {print $2}' "$EXE" |\
			tr -d "{()" | sort
		exit
	;;
	--generic|-g)
		cd "$DIR" || exit
		chk_overview
		chk_hardware
		chk_kernel
		chk_modules
		chk_multipath
		chk_storage
		chk_filesys
		chk_network
		chk_nameres
		chk_timeres
		chk_software
		chk_updates
		chk_services
		chk_sysconf
		chk_errors
		chk_sysload

		rm -rf "${TEMP}"
		exit
	;;
	--function|-f)
		cd "$DIR" || exit
		shift
		if [ $# -gt 1 ]; then	
			EXE=$(type -p "$0")
			for f in "$@"; do
				grep "^function ${f}().$" "$EXE" && $f
				shift
			done
		else $FUN
		fi

		rm -rf "${TEMP}"
		exit
	;;
	--writecfg|-w)
		writecfg
		exit
	;;
	--test|-t)
		cd "$DIR" || exit
		#set -x
		chk_sleha

		rm -rf "${TEMP}"
		exit
	;;
esac
#
