#!/bin/sh

#
# romaưå
#
export HEALTHCHK_LOG=/var/log/roma/healthchk.log
export HEALTHCHK_WATCHER_LOG=/var/log/roma/healthchk_watcher.log
export HEALTHCHK_BADLOG=/var/log/roma/healthchk.bad
echo watcher health cheking at `date` > $HEALTHCHK_WATCHER_LOG

# 2009.4 higashi
# ƥ൯ưromaưƤ뤳Ȥǧ롣
if [ -e /var/log/roma/roma_watcher_1st_flg ];
	then
		echo roma or watcher failed in the first start. >> $HEALTHCHK_WATCHER_LOG
		cat $HEALTHCHK_WATCHER_LOG >> $HEALTHCHK_BADLOG
		exit 0;
fi

# 2009.4 higashi
# roma߽ǤʤȤǧ
# - /var/log/roma/roma_chk_flgʤȤǧ롣¸ߤʤϲΤ줫
#   桼ˤäromaư줿
#   ƥ൯ư
# 5ʬʾ/var/log/roma/roma_chk_flg硢ƥإ륹åԤ
if [ -e /var/log/roma/roma_chk_flg ];
	then
		ROMACHKFLGSTAT=`stat -t /var/log/roma/roma_chk_flg | /bin/cut -d " " -f13`
		DATENOW=`/bin/date +%s`
		echo roma_chk_flg time stamp $ROMACHKFLGSTAT. >> $HEALTHCHK_WATCHER_LOG
		if [ `expr $DATENOW - $ROMACHKFLGSTAT` -ge 300 ];
			then
				rm -f /var/log/roma/roma_chk_flg
				echo roma_chk_flg is delete. >> $HEALTHCHK_WATCHER_LOG
				logger -t romachk roma_chk_flg is deleted.
			else
				echo roma is executing stop process. >> $HEALTHCHK_WATCHER_LOG
				cat $HEALTHCHK_WATCHER_LOG >> $HEALTHCHK_BADLOG
				exit 0;
		fi
fi

#2009.4 higashi
#watcherߤƤʤȤǧ
WATCHERPID=`pgrep -P 1 watcher`

# 2009.12 higashi
# roma߽ǤʤȤٳǧ
if [ -e /var/log/roma/roma_chk_flg ] ||  [ -e /var/log/roma/roma_watcher_1st_flg ];
	then
		echo roma is restart \(watcher check\). >> $HEALTHCHK_WATCHER_LOG
		cat $HEALTHCHK_WATCHER_LOG >> $HEALTHCHK_BADLOG
		exit 0;
fi

ROMACHK_FLG=0
if [ "$WATCHERPID" != "" ];
	then
		echo watcher is running. >> $HEALTHCHK_WATCHER_LOG
		echo okay. >> $HEALTHCHK_WATCHER_LOG
	else
		logger -t romachk Process of the watcher did not exist.
		echo watcher is not running. >> $HEALTHCHK_WATCHER_LOG
		ROMAPID=`pgrep -P 1 roma`
		if [ "$ROMAPID" == "" ];
			then
				ROMACHK_FLG=1
		fi
		/opt/nec/roma/bin/watcher -k start
		sleep 5
		WATCHERPID=`pgrep -P 1 watcher`
		if [ "$WATCHERPID" != "" ];
			then
				echo watcher is restart success. >> $HEALTHCHK_WATCHER_LOG
				cat $HEALTHCHK_WATCHER_LOG >> $HEALTHCHK_BADLOG
			else
				echo watcher is restart failed. >> $HEALTHCHK_WATCHER_LOG
				logger -t romachk watcher is restart failed.
				cat $HEALTHCHK_WATCHER_LOG >> $HEALTHCHK_BADLOG
				sleep 5
				echo reboot > /dev/roma
				/sbin/reboot&
				exit;
		fi
fi

#2009.4 higashi
#СϽλƻξϡromaλƻ5ʬ˹Ԥ
#watcher줵ϹԤʤ
ROMAPROXYMODE=`grep 'main_proxy_mode' /opt/nec/roma/etc/roma.conf | head -n 1 | sed 's/main_proxy_mode[^(0-9)]*\([(0-9)]*\)/\1/'`
ROMACHKMODE=`grep 'roma_healthchk_mode' /opt/nec/roma/etc/roma.conf | perl -p -i -e "s/roma_healthchk_mode\s+(0|1)/\1/"`
if [ "$ROMACHKMODE" != "0" ] && [ "$ROMACHKMODE" != "1" ];
	then
		ROMACHKMODE=0
fi
if [ $ROMAPROXYMODE == 3 ] || [ "$ROMACHKMODE" == "0" ];
	then
		NOWDATE=`/bin/date +%M`
		MINIT=`expr $NOWDATE % 5`
		if [ "$MINIT" != "0" ] || [ "$ROMACHK_FLG" != "0" ];
			then
				exit 0;
		fi
fi

NOWDATEH=`/bin/date +%H`
NOWDATEM=`/bin/date +%M`
if [ "$NOWDATEH" == "00" ] && [ "$NOWDATEM" -lt 10 ];
	then
		exit 0;
	else
		if [ "$NOWDATEH" == "04" ] && [ "$NOWDATEM" -ge 2 ] && [ "$NOWDATEM" -lt 10 ];
			then
				exit 0;
		fi
fi

echo roma health cheking at `date` > $HEALTHCHK_LOG

# ưromaIPɥ쥹/ݡֹroma.confROMAHOSTPORTسǼ
# θwgetޥɤѤƼºݤ˥Ԥ
rm -f /tmp/health-check

# forwardreversemain_http_host_portν񼰤ۤʤ뤿ʬ
if [ $ROMAPROXYMODE == 3 ]; 
	then
		#echo "roma is running in reverse proxy mode."
		ROMAHOSTPORT_=`grep '^\s*main_http_host_port' /opt/nec/roma/etc/roma.conf | grep -Ev '[[:blank:]]+443[[:blank:]]+' | head -n 1 | sed 's/main_http_host_port \([(0-9).]*\)[^(0-9)]*\([(0-9)]*\)[^(0-9)]*.*/\1:\2/'`
		ROMAHOSTPORT2_=`grep '^\s*main_http_host_port' /opt/nec/roma/etc/roma.conf | grep -Ev '[[:blank:]]+443[[:blank:]]+' | head -n 1 | sed 's/main_http_host_port \([(0-9).]*\)[^(0-9)]*\([(0-9)]*\)[^(0-9)]*.*/\1 \2/'`
		break;
	else
		#echo "roma is running in forward proxy mode."
		ROMAHOSTPORT_=`grep '^\s*main_http_host_port' /opt/nec/roma/etc/roma.conf | head -n 1 | sed 's/main_http_host_port \([(0-9).]*\)[^(0-9)]*\([(0-9)]*\)/\1:\2/'`
		ROMAHOSTPORT2_=`grep '^\s*main_http_host_port' /opt/nec/roma/etc/roma.conf | head -n 1 | sed 's/main_http_host_port \([(0-9).]*\)[^(0-9)]*\([(0-9)]*\)/\1 \2/'`
		export http_proxy=http://$ROMAHOSTPORT/
fi

#2010.03 higashi
#СץromaԤ443ΤߤȽ
if [ $ROMAPROXYMODE == 3 ] && ([ "$ROMAHOSTPORT_" == "" ] || [ "$ROMAHOSTPORT2_" == "" ])
	then
		ROMAHOSTPORT_=`grep '^\s*main_http_host_port' /opt/nec/roma/etc/roma.conf | head -n 1 | sed 's/main_http_host_port \([(0-9).]*\)[^(0-9)]*\([(0-9)]*\)[^(0-9)]*.*/\1:\2/'`
		ROMAHOSTPORT2_=`grep '^\s*main_http_host_port' /opt/nec/roma/etc/roma.conf | head -n 1 | sed 's/main_http_host_port \([(0-9).]*\)[^(0-9)]*\([(0-9)]*\)[^(0-9)]*.*/\1 \2/'`
		break;
fi

ROMAHOSTPORT=`echo $ROMAHOSTPORT_ | perl -p -i -e "s/(^|[\. \:])0+([1-9])/\1\2/g"`
ROMAHOSTPORT2=`echo $ROMAHOSTPORT2_ | perl -p -i -e "s/(^|[\. ])0+([1-9])/\1\2/g"`
#echo $ROMAHOSTPORT
#echo $ROMAHOSTPORT2

#internal objectΥץե
ROMAINTPREFX=`grep 'main_internal_objects_prefix' /opt/nec/roma/etc/roma.conf | head -n 1 | sed 's/main_internal_objects_prefix[ ]*\([^\n.]*\)/\1/'`
ROMAINTPRFXLEN=`echo $ROMAINTPREFX | wc -w`
if [ $ROMAINTPRFXLEN == 0 ];
	then
		ROMAINTPREFX="roma-internal"
fi

#echo trying URL is http://$ROMAHOSTPORT/$ROMAINTPREFX/health-check

#2009.05 higashi б
#Сץϥ⡼ɤǻƻΥѹ
TESTURL=http://$ROMAHOSTPORT/$ROMAINTPREFX/health-check
if [ $ROMAPROXYMODE != 3 ] && [ "$ROMACHKMODE" == "1" ];
	then
		WPADPS=`pgrep -P1 wpad`
		if [ "$WPADPS" != "" ];
			then
				ROMAHOST=`echo $ROMAHOSTPORT | perl -p -i -e "s/(^[0-9 .]+):[0-9]+$/\1/g"`
				ROMAHOSTPORT=$ROMAHOST:80
				TESTURL=http://$ROMAHOSTPORT/proxy.pac
			else
				ROMAHOST=`grep -E "^Listen[[:blank:]]+[0-9 .]+:[0-9]+$" /etc/opt/nec/wbmchttpd/conf/wbmchttpd.conf | head -n 1 | perl -p -i -e "s/^Listen\s+([0-9 .]+:[0-9]+)$/\1/g"`
				if [ "$ROMAHOST" == "" ];
					then
						ROMAHOST=`echo $ROMAHOSTPORT | perl -p -i -e "s/(^[0-9 .]+):[0-9]+$/\1/g"`
						ROMAPORT=`grep -E "^Port[[:blank:]]+[0-9]+$" /etc/opt/nec/wbmchttpd/conf/wbmchttpd.conf  | head -n 1 | perl -p -i -e "s/^Port\s+(.+)/\1/g"`
						ROMAHOSTPORT=$ROMAHOST:$ROMAPORT
					else
						ROMAHOSTPORT=$ROMAHOST
				fi
				TESTURL=http://$ROMAHOSTPORT/
		fi
fi


OUTFILE=/tmp/health-check

# download check file, retry 5 times, interval 10sec.
DOWNLDOK=0
WK=0
GETCHKFLG=0
while [ $WK -lt 5 ]; do
		#ꥯromachk-getĤäƤ齪λ
		pkill romachk-get >& /dev/null
        rm -f /tmp/health-check
        #wget -T 5 -w 5 -P /tmp -q http://$ROMAHOSTPORT/$ROMAINTPREFX/health-check > /dev/null 2>&1 &
        /opt/nec/roma/bin/romachk-get $ROMAHOSTPORT2 $TESTURL $OUTFILE 2>&1 &
        sleep 10
        if [ -f $OUTFILE ];
            then
                DOWNLDOK=1
                break;
            else
				echo "romachk-get faile host:`hostname -s`  date:`date`." >> $HEALTHCHK_LOG
				GETCHKFLG=1
        fi
	if [ -e /var/log/roma/roma_chk_flg ] ;
		then
			ROMACHKFLGSTAT=`stat -t /var/log/roma/roma_chk_flg | /bin/cut -d " " -f13`
			echo roma is restart \(roma check\).  roma_chk_flg time stamp $ROMACHKFLGSTAT >> $HEALTHCHK_LOG
			cat $HEALTHCHK_LOG >> $HEALTHCHK_BADLOG
			exit 0;
	elif [ -e /var/log/roma/roma_watcher_1st_flg ];
		then
			echo roma is restart \(roma check\). >> $HEALTHCHK_LOG
			cat $HEALTHCHK_LOG >> $HEALTHCHK_BADLOG
			exit 0;
	fi
        ((WK++))
        #echo wget: timeout occured, retry
done

ADMIN=$3
# 餫Υե뤬Ǥɤǧ
if [ $DOWNLDOK == '1' ];
	then
		# exists. do nothing.
		echo roma is running. >> $HEALTHCHK_LOG
		echo okay. >> $HEALTHCHK_LOG
		#1Ǥƻ˼ԤĤ
		if [ "$GETCHKFLG" == "1" ];
			then
				cat $HEALTHCHK_LOG >> $HEALTHCHK_BADLOG
		fi
		break;
	else
		logger -t romachk Response was not returned from roma.
		/bin/touch /var/log/roma/icpcheck.status
		echo "roma of `hostname -s` may be dead at `date`." >> $HEALTHCHK_LOG
		#echo "roma of `hostname -s` may be dead at `date`." | mail -s "notice of `hostname -s`" $ADMIN
		if [ $ROMAPROXYMODE == 3 ] || [ "$ROMACHKMODE" == "0" ];
			then
				/usr/sbin/tcpdump -s 1500 -w /var/log/roma/tcpdump.dat -i any -c 300 >& /var/tmp/romachk.log &
		fi
		/bin/tar -czvlf /var/log/roma/roma_errs.tar.gz /var/log/roma/roma_err.* >& /var/tmp/romachk.log
		netstat -n > /var/log/roma/netstat-n.txt
		netstat -s > /var/log/roma/netstat-s.txt
		ps -ax > /var/log/roma/ps-ax.txt
		vmstat 1 1 > /var/log/roma/vmstat.txt
		top b n 1 > /var/log/roma/top.txt
		tail -n 100 /var/log/roma/access_log > /var/log/roma/lastaccess.txt
		tail -n 100 /var/log/roma/roma_log > /var/log/roma/lastromalog.txt
		tail -n 100 /var/log/messages > /var/log/roma/lastmessages.txt
		# 2009.06 higashi
		# Ѥ˰,Сȥ⡼ɤoffξϽɤΥåԤ
		if [ $ROMAPROXYMODE == 3 ] || [ "$ROMACHKMODE" == "0" ];
			then
				pkill -USR2 -P 1 roma >& /dev/null
				if [ "$?" = "0" ];
					then
						echo roma is running. >> $HEALTHCHK_LOG
						CNT=0
						rm -f /var/log/roma/roma_md_chk_flg > /dev/null 2>&1
						while [ $CNT -le 60 ];do
							if [ -e /var/log/roma/roma_md_chk_flg ];
								then
									break;
							fi
							((CNT++))
							sleep 1
						done
						if [ $CNT -ge 60 ];then
							logger -t romachk Md file was not able to be writen within one minute.
						fi
						pkill -TRAP -P 1 roma >& /dev/null
						break;
					else
						echo roma is not running. >> $HEALTHCHK_LOG
						cat $HEALTHCHK_LOG >> $HEALTHCHK_BADLOG
						exit 0;
				fi
				cat $HEALTHCHK_LOG >> $HEALTHCHK_BADLOG
				break;
			else
				cat $HEALTHCHK_LOG >> $HEALTHCHK_BADLOG
				sleep 5
				echo reboot > /dev/roma
				/sbin/reboot&
		fi
fi
