#! /bin/bash

# constants
WATCH_DAT="/etc/iplb4.d/watch.dat"
WAIT_DAT="/etc/iplb4.d/wait.dat"
CHANGE_COOKIE_DAT="/etc/iplb4.d/change_cookie_time_up.dat"
WATCH_LOG="/var/log/iplb4/watch.log"
WATCH_BAK_LOG="/var/log/iplb4/watch.bak" # Add 2009/02/26 by Y.Shiraishi
LOG_DIR=`dirname $WATCH_LOG`
LOCK_FILE="/tmp/.iplbwatch-lock"
LOCK_TMP="/tmp/.iplbshell.tmp$$"
RESTART_FLAG_FILE="/tmp/iplbrestart"

#CONF="/etc/ha4.d/iplb4.conf"
CGI_MONITOR="/etc/iplb4.d/cgi-monitor"   # Add 2009/02/25 by Y.Shiraishi
SERVER_TYPE="IPLBConf"                   # Add 2009/02/25 by Y.Shiraishi
INITD="/etc/rc.d/init.d/iplb4d"
DAEMON="/usr/lib/iplb4/iplb4d"
DAEMON_NAME="iplb4d"			 # 2006/02/10 by T.Uenosono  ץID˻
SENDARP="/usr/lib/iplb4/send_arp"

# CONT_CONF="/etc/iplb_contents/contents.conf"
CONT_INITD="/etc/rc.d/init.d/cntsd"
CONT_DAEMON="/usr/lib/iplb_contents/cntsd"
CONT_DAEMON_NAME="cntsd"		 # 2006/02/10 by T.Uenosono  ץID˻
CONT_FTPCOPY="/usr/bin/ftpcopy"
CONT_LBPOST="/usr/bin/lbpost"
CONT_LBSYNC="/usr/bin/lbsync"

MASTER_DIR="/usr/lib/iplb4/f-master"

#CONF_MASTER="conf.master"
INITD_MASTER="initd.master"
DAEMON_MASTER="iplb4d.master"
SENDARP_MASTER="send_arp.master"

# CONT_CONF_MASTER="contconf.master"
CONT_INITD_MASTER="continitd.master"
CONT_DAEMON_MASTER="cntsd.master"
CONT_FTPCOPY_MASTER="ftpcopy.master"
CONT_LBPOST_MASTER="lbpost.master"
CONT_LBSYNC_MASTER="lbsync.master"

DAEMON_DFLAG="0"
CONT_DAEMON_DFLAG="0"
AUTO_REBOOT_FLAG="0"		# 2009/06/08 by Y.Shiraishi ưƵư(0=ͭ,1=̵)

DAEMON_MIN="7"
CONT_DAEMON_MIN="5"
RESTART_DAEMON_CMD="/etc/rc.d/init.d/iplb4d restart "
RESTART_CONT_DAEMON_CMD="/etc/rc.d/init.d/cntsd restart "
RESTART_WAIT="10"                       # Add 2009/02/27 by Y.Shiraishi 5ä10äѹ
SLEEP_MIN="1"
SLEEP_MAX="1440"

# set default value to parameters 
PROCESS_M=0
PROCESS_R=0
FILE_M=0
FILE_R=0
FO_M=0
FO_R=0
FB_M=0
FB_R=0
LBH_DOWN_M=0
LBH_DOWN_R=0
MAIL_ADDR1=""
MAIL_SERVER1=""
MAIL_ADDR2=""
MAIL_SERVER2=""
SECONDARY_ENABLE=0
SLEEP_TIME="1"
PROCESS_RECOV=0

# trap
trap 'release_lock; rm -f $LOCK_FILE $LOCK_TMP >/dev/null 2>&1; exit 0' 0 1 2 3 15

# functions
get_lock() {

	# create local lock file.
	#if [ ! -e $LOCK_TMP ]
	#then
		echo $$ > $LOCK_TMP
	#fi

	# check and create lock file in atomic action.
	until ln -s $LOCK_TMP $LOCK_FILE 2>/dev/null
	do
		sleep 1
	done
}

release_lock() {
	if [ -e $LOCK_FILE ] && [ $$ = `cat $LOCK_FILE` ]
	then
		rm -f $LOCK_FILE > /dev/null 2>&1
	fi
}

write_event() {
	get_lock
	date "+%Y%m%d%H%M%S : $*" >> $WATCH_LOG
	release_lock
}

# Add 2009/02/26 by Y.Shiraishi
write_event2() {
        get_lock
        date "+%Y%m%d%H%M%S : $*" >> $WATCH_BAK_LOG
	release_lock
}

process_check() {

# count instances of iplb4d.
	DAEMON_NUM=`/sbin/pidof $DAEMON_NAME | wc -w `
	
	if [ "$DAEMON_NUM" -lt "$DAEMON_MIN" ]
	then
		if [ "${DAEMON_DFLAG}" = "0" ]
		then
			write_event "Process error"
			# Add 2006/02/10 by Uenosono The write-in function to syslog.
			logger "LBtrap(1) : IPLB daemon Process Error" > /dev/null 2>&1
			DAEMON_DFLAG=1
		fi

	# if you use following commented code.
	# you must think deeply about RESTART_WAIT.

		if [ "${PROCESS_RECOV}" = "1" ]
		then
			$RESTART_DAEMON_CMD > /dev/null 2>&1
			sleep $RESTART_WAIT
			DAEMON_NUM=`/sbin/pidof $DAEMON_NAME | wc -w `
			if [ "$DAEMON_NUM" -lt "$DAEMON_MIN" ]
			then
				write_event "Process recover error"
			else
				write_event "Process recover"
				# Add 2006/02/10 by Uenosono The write-in function to syslog.
				logger "LBtrap(2) : IPLB daemon Process Recover" > /dev/null 2>&1
				DAEMON_DFLAG=0
			fi
		fi

		# Add 2009/03/03 by Y.Shiraishi
		# ưƵư꤬դξǤ⡢Хååץǥ͡ξ
		# iplb4dץ塢̵iplb4dƵư
		if [ "$AUTO_REBOOT_FLAG" = "0" -a "${PROCESS_RECOV}" = "0" ]
		then
			while read FIELD_ITEM FIELD_VALUE
			do
				if [ "$FIELD_ITEM" = "$SERVER_TYPE" -a "$FIELD_VALUE" = "Backup" ]
				then
					$RESTART_DAEMON_CMD > /dev/null 2>&1
					sleep $RESTART_WAIT
					DAEMON_NUM=`/sbin/pidof $DAEMON_NAME | wc -w `
					write_event2 "iplb4d Process Count $DAEMON_NUM"
					if [ "$DAEMON_NUM" -lt "$DAEMON_MIN" ]
					then
						write_event2 "Process recover error"
					else
						write_event2 "Process recover"
						# Add 2006/02/10 by Uenosono The write-in function to syslog.
						logger "LBtrap(2) : IPLB daemon Process Recover" > /dev/null 2>&1
						DAEMON_DFLAG=0
					fi
				fi
				break
			done < $CGI_MONITOR
		fi
	else
		DAEMON_DFLAG=0
	fi

# count instances of contentsd.
	if test -e $CONT_DAEMON
	then
	    DAEMON_NUM=`/sbin/pidof $CONT_DAEMON_NAME | wc -w `
	    
	    if [ "$DAEMON_NUM" -lt "$CONT_DAEMON_MIN" ]
	    then
	    	if [ "${CONT_DAEMON_DFLAG}" = "0" ]
	    	then
	    		write_event "Contents Process error"
	    		CONT_DAEMON_DFLAG=1
	    	fi

	    # if you use following commented code.
	    # you must think deeply about RESTART_WAIT.

	    	if [ "${PROCESS_RECOV}" = "1" ]
	    	then
	    		$RESTART_CONT_DAEMON_CMD > /dev/null 2>&1
	    		sleep $RESTART_WAIT
	    		DAEMON_NUM=`/sbin/pidof $CONT_DAEMON_NAME | wc -w `
	    		if [ "$DAEMON_NUM" -lt "$CONT_DAEMON_MIN" ]
	    		then
	    			write_event "Contents Process recover error"
	    		else
	    			write_event "Contents Process recover"
	    			CONT_DAEMON_DFLAG=0
	    		fi
	    	fi
	    else
	    	CONT_DAEMON_DFLAG=0
	    fi
	fi

	# /tmp/iplbrestart ե뤬¸ߤƤ硢
	# Ʊե塢iplb4dƵư
	if [ -e $RESTART_FLAG_FILE ]
	then
		$RESTART_DAEMON_CMD > /dev/null 2>&1
		sleep $RESTART_WAIT
		DAEMON_NUM=`/sbin/pidof $DAEMON_NAME | wc -w `
		if [ "$DAEMON_NUM" -lt "$DAEMON_MIN" ]
		then
			write_event "Process recover error"
		else
			write_event "Process recover"
			logger "LBtrap(2) : IPLB daemon Process Recover" > /dev/null 2>&1
			rm -f $RESTART_FLAG_FILE > /dev/null 2>&1
			DAEMON_DFLAG=0
		fi
	fi

}

file_check() {

	# check status of files.
#	CONF_LS_TMP=`ls -l --full-time $CONF 2>/dev/null`
	INITD_LS_TMP=`ls -l --full-time $INITD 2>/dev/null`
	DAEMON_LS_TMP=`ls -l --full-time $DAEMON 2>/dev/null`
	SENDARP_LS_TMP=`ls -l --full-time $SENDARP 2>/dev/null`

	# check contents status of files.
#	CONT_CONF_LS_TMP=`ls -l --full-time $CONT_CONF 2>/dev/null`
	if test -e $CONT_INITD
	then
		CONT_INITD_LS_TMP=`ls -l --full-time $CONT_INITD 2>/dev/null`
	fi
	if test -e $CONT_DAEMON
	then
		CONT_DAEMON_LS_TMP=`ls -l --full-time $CONT_DAEMON 2>/dev/null`
	fi
	if test -e $CONT_FTPCOPY
	then
		CONT_FTPCOPY_LS_TMP=`ls -l --full-time $CONT_FTPCOPY 2>/dev/null`
	fi
	if test -e $CONT_LBPOST
	then
		CONT_LBPOST_LS_TMP=`ls -l --full-time $CONT_LBPOST 2>/dev/null`
	fi
	if test -e $CONT_LBPOST
	then
		CONT_LBSYNC_LS_TMP=`ls -l --full-time $CONT_LBSYNC 2>/dev/null`
	fi

	# compare current status with stored status.
#	if [ "$CONF_LS_TMP" != "$CONF_LS" ]
#	then
#		write_event "File error $CONF"
#		cp -pf ${MASTER_DIR}/${CONF_MASTER} $CONF 2>/dev/null
#
#		CONF_LS_TMP=`ls -l --full-time $CONF 2>/dev/null`
#		if [ "$CONF_LS_TMP" = "$CONF_LS" ]
#		then
#			write_event "File recover $CONF"
#		else
#			write_event "File recover error $CONF"
#		fi
#	fi

	if [ "$INITD_LS_TMP" != "$INITD_LS" ]
	then
		write_event "File error $INITD"
		# Add 2006/02/10 by Uenosono The write-in function to syslog.
		logger "LBtrap(3) : IPLB File($INITD) Error" > /dev/null 2>&1
		cp -pf ${MASTER_DIR}/${INITD_MASTER} $INITD 2>/dev/null

		INITD_LS_TMP=`ls -l --full-time $INITD 2>/dev/null`
		if [ "$INITD_LS_TMP" = "$INITD_LS" ]
		then
			write_event "File recover $INITD"
			# Add 2006/02/10 by Uenosono The write-in function to syslog.
			logger "LBtrap(4) : IPLB File($INITD) Recover" > /dev/null 2>&1
		else
			write_event "File recover error $INITD"
		fi
	fi

	if [ "$DAEMON_LS_TMP" != "$DAEMON_LS" ]
	then
		write_event "File error $DAEMON"
		# Add 2006/02/10 by Uenosono The write-in function to syslog.
		logger "LBtrap(3) : IPLB File($DAEMON) Error" > /dev/null 2>&1
		cp -pf ${MASTER_DIR}/${DAEMON_MASTER} $DAEMON 2>/dev/null

		DAEMON_LS_TMP=`ls -l --full-time $DAEMON 2>/dev/null`
		if [ "$DAEMON_LS_TMP" = "$DAEMON_LS" ]
		then
			write_event "File recover $DAEMON"
			# Add 2006/02/10 by Uenosono The write-in function to syslog.
			logger "LBtrap(4) : IPLB File($DAEMON) Recover" > /dev/null 2>&1
		else
			write_event "File recover error $DAEMON"
		fi
	fi

	if [ "$SENDARP_LS_TMP" != "$SENDARP_LS" ]
	then
		write_event "File error $SENDARP"
		# Add 2006/02/10 by Uenosono The write-in function to syslog.
		logger "LBtrap(3) : IPLB File($SENDARP) Error" > /dev/null 2>&1
		cp -pf ${MASTER_DIR}/${SENDARP_MASTER} $SENDARP 2>/dev/null

		SENDARP_LS_TMP=`ls -l --full-time $SENDARP 2>/dev/null`
		if [ "$SENDARP_LS_TMP" = "$SENDARP_LS" ]
		then
			write_event "File recover $SENDARP"
			# Add 2006/02/10 by Uenosono The write-in function to syslog.
			logger "LBtrap(4) : IPLB File($SENDARP) Recover" > /dev/null 2>&1
		else
			write_event "File recover error $SENDARP"
		fi
	fi

	if test -e $CONT_DAEMON
	then
		if [ "$CONT_DAEMON_LS_TMP" != "$CONT_DAEMON_LS" ]
		then
			write_event "File error $CONT_DAEMON"
			cp -pf ${MASTER_DIR}/${CONT_DAEMON_MASTER} $CONT_DAEMON 2>/dev/null

			CONT_DAEMON_LS_TMP=`ls -l --full-time $CONT_DAEMON 2>/dev/null`
			if [ "$CONT_DAEMON_LS_TMP" = "$CONT_DAEMON_LS" ]
			then
				write_event "File recover $CONT_DAEMON"
			else
				write_event "File recover error $CONT_DAEMON"
			fi
		fi
	fi

#	if [ "$CONT_CONF_LS_TMP" != "$CONT_CONF_LS" ]
#	then
#		write_event "File error $SENDARP"
#		cp -pf ${MASTER_DIR}/${CONT_CONF_MASTER} $CONT_CONF 2>/dev/null
#
#		CONT_CONF_LS_TMP=`ls -l --full-time $CONT_CONF 2>/dev/null`
#		if [ "$CONT_CONF_LS_TMP" = "$CONT_CONF_LS" ]
#		then
#			write_event "File recover $CONT_CONF"
#		else
#			write_event "File recover error $CONT_CONF"
#		fi
#	fi
	if test -e $CONT_INITD
	then
		if [ "$CONT_INITD_LS_TMP" != "$CONT_INITD_LS" ]
		then
			write_event "File error $CONT_INITD"
			cp -pf ${MASTER_DIR}/${CONT_INITD_MASTER} $CONT_INITD 2>/dev/null

			CONT_INITD_LS_TMP=`ls -l --full-time $CONT_INITD 2>/dev/null`
			if [ "$CONT_INITD_LS_TMP" = "$CONT_INITD_LS" ]
			then
				write_event "File recover $CONT_INITD"
			else
				write_event "File recover error $CONT_INITD"
			fi
		fi
	fi

	if test -e $CONT_FTPCOPY
	then
		if [ "$CONT_FTPCOPY_LS_TMP" != "$CONT_FTPCOPY_LS" ]
		then
			write_event "File error $CONT_FTPCOPY"
			cp -pf ${MASTER_DIR}/${CONT_FTPCOPY_MASTER} $CONT_FTPCOPY 2>/dev/null

			CONT_FTPCOPY_LS_TMP=`ls -l --full-time $CONT_FTPCOPY 2>/dev/null`
			if [ "$CONT_FTPCOPY_LS_TMP" = "$CONT_FTPCOPY_LS" ]
			then
				write_event "File recover $CONT_FTPCOPY"
			else
				write_event "File recover error $CONT_FTPCOPY"
			fi
		fi
	fi

	if test -e $CONT_LBPOST
	then
		if [ "$CONT_LBPOST_LS_TMP" != "$CONT_LBPOST_LS" ]
		then
			write_event "File error $CONT_LBPOST"
			cp -pf ${MASTER_DIR}/${CONT_LBPOST_MASTER} $CONT_LBPOST 2>/dev/null

			CONT_LBPOST_LS_TMP=`ls -l --full-time $CONT_LBPOST 2>/dev/null`
			if [ "$CONT_LBPOST_LS_TMP" = "$CONT_LBPOST_LS" ]
			then
				write_event "File recover $CONT_LBPOST"
			else
				write_event "File recover error $CONT_LBPOST"
			fi
		fi
	fi

	if test -e $CONT_LBPOST
	then
		if [ "$CONT_LBSYNC_LS_TMP" != "$CONT_LBSYNC_LS" ]
		then
			write_event "File error $CONT_LBSYNC"
			cp -pf ${MASTER_DIR}/${CONT_LBSYNC_MASTER} $CONT_LBSYNC 2>/dev/null

			CONT_LBSYNC_LS_TMP=`ls -l --full-time $CONT_LBSYNC 2>/dev/null`
			if [ "$CONT_LBSYNC_LS_TMP" = "$CONT_LBSYNC_LS" ]
			then
				write_event "File recover $CONT_LBSYNC"
			else
				write_event "File recover error $CONT_LBSYNC"
			fi
		fi
	fi
}

file_backup() {

	# if backup dir does not exist. create it.
	if [ ! -d $MASTER_DIR ]
	then
		rm -rf $MASTER_DIR
		mkdir -p $MASTER_DIR
	fi

	# store status of files.
#	CONF_LS=`ls -l --full-time $CONF 2>/dev/null`
	INITD_LS=`ls -l --full-time $INITD 2>/dev/null`
	DAEMON_LS=`ls -l --full-time $DAEMON 2>/dev/null`
	SENDARP_LS=`ls -l --full-time $SENDARP 2>/dev/null`

	# store contents store ststus files.
	if test -e $CONT_DAEMON
	then
		CONT_DAEMON_LS=`ls -l --full-time $CONT_DAEMON 2>/dev/null`
	fi
#	CONT_CONF_LS=`ls -l --full-time $CONT_CONF 2>/dev/null`
	if test -e $CONT_INITD
	then
		CONT_INITD_LS=`ls -l --full-time $CONT_INITD 2>/dev/null`
	fi
	if test -e $CONT_FTPCOPY
	then
		CONT_FTPCOPY_LS=`ls -l --full-time $CONT_FTPCOPY 2>/dev/null`
	fi
	if test -e $CONT_LBPOST
	then
		CONT_LBPOST_LS=`ls -l --full-time $CONT_LBPOST 2>/dev/null`
	fi
	if test -e $CONT_LBPOST
	then
		CONT_LBSYNC_LS=`ls -l --full-time $CONT_LBSYNC 2>/dev/null`
	fi

	# backup files to backup dir.
	# assume these files are correct at first.
	#cp -pf $CONF ${MASTER_DIR}/${CONF_MASTER} 2>/dev/null
	cp -pf $INITD ${MASTER_DIR}/${INITD_MASTER} 2>/dev/null
	cp -pf $DAEMON ${MASTER_DIR}/${DAEMON_MASTER} 2>/dev/null
	cp -pf $SENDARP ${MASTER_DIR}/${SENDARP_MASTER} 2>/dev/null

	# backup contents files to backup dir.
	# assume these files are correct at first.
	# cp -pf $CONT_CONF ${MASTER_DIR}/${CONT_CONF_MASTER} 2>/dev/null
	if test -e $CONT_INITD
	then
		cp -pf $CONT_INITD ${MASTER_DIR}/${CONT_INITD_MASTER} 2>/dev/null
	fi
	if test -e $CONT_DAEMON
	then
		cp -pf $CONT_DAEMON ${MASTER_DIR}/${CONT_DAEMON_MASTER} 2>/dev/null
	fi
	if test -e $CONT_FTPCOPY
	then
	cp -pf $CONT_FTPCOPY ${MASTER_DIR}/${CONT_FTPCOPY_MASTER} 2>/dev/null
	fi
	if test -e $CONT_LBPOST
	then
		cp -pf $CONT_LBPOST ${MASTER_DIR}/${CONT_LBPOST_MASTER} 2>/dev/null
	fi
	if test -e $CONT_LBPOST
	then
		cp -pf $CONT_LBSYNC ${MASTER_DIR}/${CONT_LBSYNC_MASTER} 2>/dev/null
	fi
}

#
# main 
#
proc_flg=0
proc_flg2=0
sleep_flg=0
proc_sleep_time=30
# initialize
rm -f $LOCK_FILE /tmp/.iplbshell.tmp* 
rm -f $RESTART_FLAG_FILE

# backup files.
file_backup

# if backup dir does not exist. create it.
if [ ! -d $LOG_DIR ]
then
	rm -rf $LOG_DIR 2>/dev/null
	mkdir -p $LOG_DIR
fi

# if configuration file does not exist. exit.
if [ ! -s $WATCH_DAT ]
then
	echo "$WATCH_DAT does not exist."
	exit 1
fi

sleep $RESTART_WAIT

# main loop
while true
do

	# if configuration file was removed suddenly,
	# use last value.
	if [ -s $WATCH_DAT ]
	then
		# read configuration file and set parameters.
		{
			read PROCESS_M  PROCESS_R \
			     FILE_M     FILE_R \
			     FO_M       FO_R \
			     FB_M       FB_R \
			     LBH_DOWN_M LBH_DOWN_R \
			     MAIL_ADDR1 \
			     MAIL_SERVER1 \
			     MAIL_ADDR2 \
			     MAIL_SERVER2 \
			     SECONDARY_ENABLE \
			     SLEEP_TIME \
			     PROCESS_RECOV

		} < $WATCH_DAT
	fi

	if [ $proc_flg = "0" ]
	then
		if [ -s $WAIT_DAT ]
		then
			proc_flg=1
			# read configuration file and set parameters.
			{
				read REQ_WAIT_TIME
			} < $WAIT_DAT
			
			NUMERIC_CHECK=`echo "${REQ_WAIT_TIME}" | awk '/[^0-9]/ {print}'`
			
			if [[ ! -z $NUMERIC_CHECK ]]
			then
				proc_flg=2
			fi
		fi
		
		if [ $proc_flg = "1" ]
		then
			if [ $sleep_flg = "0" ]
				then
				sleep $proc_sleep_time
				sleep_flg=1
			fi
			echo ${REQ_WAIT_TIME} > /proc/sys/net/ktcpvs/change_req_wait_time 2>/dev/null
			proc_flg=2
		fi
	fi

	if [ $proc_flg2 = "0" ]
	then
		if [ -s $CHANGE_COOKIE_DAT ]
		then
			proc_flg2=1
			# read configuration file and set parameters.
			{
				read CHANGE_COOKIE
			} < $CHANGE_COOKIE_DAT
			
			NUMERIC_CHECK=`echo "${CHANGE_COOKIE}" | awk '/[^0-9]/ {print}'`
			
			if [[ ! -z $NUMERIC_CHECK ]]
			then
				proc_flg2=2
			fi
		fi
		
		if [ $proc_flg2 = "1" ]
		then
			if [ $sleep_flg = "0" ]
				then
				sleep $proc_sleep_time
				sleep_flg=1
			fi
			echo ${CHANGE_COOKIE} > /proc/sys/net/ktcpvs/change_cookie_time_update_mode 2>/dev/null
			proc_flg2=2
		fi
	fi

	if [ "$SLEEP_TIME" -lt "$SLEEP_MIN" ]
	then
		SLEEP_TIME=$SLEEP_MIN
	elif [ "$SLEEP_TIME" -gt "$SLEEP_MAX" ]
	then
		SLEEP_TIME=$SLEEP_MAX
	fi

	sleep ${SLEEP_TIME}m

	if [ "${FILE_M}" = "1" ]
	then
		file_check
	fi

	if [ "${PROCESS_M}" = "1" ]
	then
		process_check
	fi
done
