#!/bin/sh

# HP-UX Infrastructure Agent Patch Script (POSIX sh compatible)

. "$1"
export BACKUP_DIR=/tmp
export LOG_FILE=/var/log/whatap_update.log
export PATCH_RESULT_FILE="$INSTALL_DIR/patch_history.log"
# SERVER-1931: PATCH_MARKER_WRITTEN goes to 1 once the in-progress marker is on
# disk. Before that there is nothing to close out and PATCH_VERSION /
# PATCH_START_TIME are not set yet, so error_exit has to stay quiet.
# PATCH_RESULT_WRITTEN goes to 1 once any final record lands, so a later
# error_exit cannot append a second verdict on top of it.
PATCH_MARKER_WRITTEN=0
PATCH_RESULT_WRITTEN=0
CONF_FILE="$INSTALL_DIR/conf/whatap.conf"
CONF_BACKUP="$BACKUP_DIR/whatap.conf"

if [ -z "${pkg:-}" ]; then
	echo "usage: $0 <config-file>"
	exit 2
fi

# SERVER-1901: INSTALL_DIR scopes the leftover-process sweep in stop_service().
# An empty value there would widen the match from "<install>/whatap_infrad" to
# "/whatap_infrad" and hit agents of other install roots, so refuse to run
# without it rather than sweeping blind.
if [ -z "${INSTALL_DIR:-}" ]; then
	echo "INSTALL_DIR is not set in $1"
	exit 2
fi

log() {
	echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" | tee -a "$LOG_FILE"
}

error_exit() {
	log "ERROR: $1"
	# SERVER-1931: close out the in-progress marker before bailing. Left open it
	# stays the last line of patch_history.log forever, the agent keeps answering
	# "in progress", and OPM can only give up through max_check_count (~6m40s on
	# unix) with a generic timeout - throwing away the reason we already have
	# right here. Reachable from stop_service / backup_whatap / backup_config /
	# rollback_whatap, all of which run after the marker is written.
	if [ "$PATCH_MARKER_WRITTEN" = "1" ] && [ "$PATCH_RESULT_WRITTEN" = "0" ]; then
		write_patch_result "failed" "$PATCH_VERSION" "$PATCH_START_TIME" "$1"
	fi
	exit 1
}

error() {
	log "ERROR: $1"
}

write_patch_result() {
	status="$1"
	patch_version="$2"
	start_time="$3"
	error_message="$4"
	end_time=$(date '+%Y-%m-%d %H:%M:%S')

	# SERVER-1931: rollback_whatap clears $INSTALL_DIR before restoring it, so a
	# rollback that dies half-way can leave the directory holding
	# patch_history.log missing. Recreate it rather than lose the record.
	result_dir=$(dirname "$PATCH_RESULT_FILE")
	[ -d "$result_dir" ] || mkdir -p "$result_dir" 2>/dev/null

	if echo "$status|$patch_version|$start_time|$end_time|$error_message" >> "$PATCH_RESULT_FILE"; then
		PATCH_RESULT_WRITTEN=1
		log "Patch result written to $PATCH_RESULT_FILE"
	else
		log "ERROR: failed to write patch result to $PATCH_RESULT_FILE"
	fi
}

# SERVER-1931: append an in-progress marker at patch start so that the last
# line of patch_history.log always belongs to the current patch. Without it,
# a status query arriving before the final result is written would read the
# previous patch result (e.g. report success for a failing patch). The agent
# treats the "inprogress" status as "no final result yet" and does not report
# it to the server. Same record format, with end_time/error_message empty.
write_patch_inprogress() {
	if echo "inprogress|$PATCH_VERSION|$PATCH_START_TIME||" >> "$PATCH_RESULT_FILE"; then
		PATCH_MARKER_WRITTEN=1
		log "Patch in-progress marker written to $PATCH_RESULT_FILE"
	else
		# Nothing else in this script depends on the marker, so keep patching -
		# but say so loudly: without it a status query during this patch reads
		# the previous result again, which is what the marker exists to prevent.
		log "ERROR: failed to write in-progress marker to $PATCH_RESULT_FILE; a status query during this patch may report the previous patch result"
	fi
}

get_patch_version() {
	pkg_file="$1"
	version=""

	# Try extracting to temp dir and running the binary
	tmpdir="/tmp/whatap_ver.$$"
	rm -rf "$tmpdir" 2>/dev/null
	mkdir -p "$tmpdir" 2>/dev/null
	if [ -d "$tmpdir" ]; then
		gunzip -c "$pkg_file" 2>/dev/null | tar xf - -C "$tmpdir" 2>/dev/null
		if [ -x "$tmpdir/whatap-infra/whatap_infrad" ]; then
			version=$("$tmpdir/whatap-infra/whatap_infrad" version 2>/dev/null | head -1)
		fi
		rm -rf "$tmpdir"
	fi

	# Fallback: extract from filename
	if [ -z "$version" ]; then
		version=$(basename "$pkg_file" | sed 's/.*-\([0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\).*/\1/')
		if [ "$version" = "$(basename "$pkg_file")" ]; then
			version=$(basename "$pkg_file" | sed 's/.*-\([0-9][0-9]*\.[0-9][0-9]*\)-\([0-9][0-9]*\).*/\1.\2/')
			if [ "$version" = "$(basename "$pkg_file")" ]; then
				version="unknown"
			fi
		fi
	fi

	echo "$version"
}

backup_config() {
	log "backing up config..."
	if [ -f "$CONF_FILE" ]; then
		cp -p "$CONF_FILE" "$CONF_BACKUP" || error_exit "failed to backup config"
		log "config backed up to $CONF_BACKUP"
	else
		log "no config file found to backup"
	fi
}

restore_config() {
	log "restoring config..."
	if [ -f "$CONF_BACKUP" ]; then
		cp -p "$CONF_BACKUP" "$CONF_FILE" || error "failed to restore config"
		log "config restored from $CONF_BACKUP"
	else
		log "no config backup found to restore"
	fi
}

stop_service() {
	log "service stop..."

	PID_FILE="$INSTALL_DIR/conf/whatap.pid"
	SAVED_PID=""
	if [ -f "$PID_FILE" ]; then
		SAVED_PID=$(cat "$PID_FILE" 2>/dev/null | tr -d ' \t\r\n')
		log "pre-stop pid from $PID_FILE: $SAVED_PID"
	else
		log "$PID_FILE not found"
	fi

	/sbin/init.d/whatap-infra stop
	if [ $? -ne 0 ]; then
		log "service stop error"
	fi
	sleep 3

	# SERVER-1901: the pid file only tracks the master process. The master forks
	# a "foreground" worker child whose pid is never recorded, so stopping only
	# the master (especially with kill -9, which bypasses the agent's child-kill
	# handler) leaves the child orphaned. Orphans from earlier cycles have also
	# been reparented to init, so ppid matching cannot find them. Across repeated
	# patches these leftover workers accumulate as zombie/orphan processes on
	# HP-UX. Reap every remaining whatap_infrad process of this install.
	#
	# The match is an exact whitespace-delimited token equal to
	# "$INSTALL_DIR/whatap_infrad" - not the bare binary name and not a
	# substring. The install root is configurable, so a bare-name match would
	# also kill agents of a different install root on the same host and any
	# unrelated process that merely mentions the binary; and a substring match
	# would additionally hit the "whatap_infrad.bak-<timestamp>" copies this very
	# script leaves beside the binary.
	#
	# This sweep also replaces the old SAVED_PID escalation that used to sit
	# above it. That block matched with "ps -p <pid> | grep -q whatap_infrad" -
	# the same bare-name substring this PR removes everywhere else - and went
	# straight to kill -9, which bypasses the agent's child-kill handler and
	# orphans the worker: exactly the bug being fixed here. The sweep signals
	# SIGTERM first (handler runs, child dies with it) and only escalates to
	# SIGKILL for survivors. The abort-the-patch check that block provided is
	# kept below, scoped to the tracked master.
	FG_BIN="$INSTALL_DIR/whatap_infrad"

	leftover_pids() {
		ps -ef 2>/dev/null | awk -v bin="$FG_BIN" '
			{ for (i = 1; i <= NF; i++) if ($i == bin) { print $2; break } }'
	}

	# Poll rather than sleep a fixed amount: the agent's SIGTERM handler kills
	# its child and then sleeps 5s before exiting, so a shorter wait would
	# SIGKILL the master mid-shutdown and lose its termination log, while a flat
	# long sleep would slow down every patch. Common case ~1s, ceiling 10s.
	wait_leftover_exit() {
		waited=0
		while [ "$waited" -lt 10 ]; do
			if [ -z "$(leftover_pids)" ]; then
				return 0
			fi
			sleep 1
			waited=$((waited + 1))
		done
		return 1
	}

	# Each pid is numeric-validated so a truncated/corrupt ps line can never turn
	# kill into a broadcast (kill -1).
	for FG_PID in $(leftover_pids); do
		case "$FG_PID" in
			''|*[!0-9]*) continue ;;
		esac
		log "leftover whatap_infrad process pid $FG_PID found, terminating"
		kill -TERM "$FG_PID" 2>/dev/null
	done

	if ! wait_leftover_exit; then
		for FG_PID in $(leftover_pids); do
			case "$FG_PID" in
				''|*[!0-9]*) continue ;;
			esac
			log "process pid $FG_PID still alive, sending SIGKILL"
			kill -9 "$FG_PID" 2>/dev/null
		done
		sleep 3
	fi

	# SERVER-1901: preserve the previous abort-on-undead-master behaviour, but
	# decide it with the scoped exact-token matcher instead of a bare-name
	# substring. Scoped to the tracked master only, as before: a leftover worker
	# that survives SIGKILL should not by itself abort the patch.
	if [ -n "$SAVED_PID" ] && leftover_pids | grep -x "$SAVED_PID" >/dev/null 2>&1; then
		error_exit "pid $SAVED_PID still alive after SIGTERM and SIGKILL, error"
	fi

	log "service stop ok"
}

start_service() {
	log "service start..."
	rm -f "$INSTALL_DIR/conf/whatap_infrad.pid"
	/sbin/init.d/whatap-infra start
	if [ $? -ne 0 ]; then
		log "service start error"
	fi
	log "service start ok"
}

backup_whatap() {
	log "creating backup of existing installation..."
	if [ -d "$INSTALL_DIR" ]; then
		mkdir -p "$BACKUP_DIR"
		# cp -Rp cannot copy socket files (e.g. whatap_infrad.sock) - ignore those errors
		cp -Rp "$INSTALL_DIR" "$BACKUP_DIR/" 2>/dev/null || true
		if [ ! -d "$BACKUP_DIR/infra" ]; then
			error_exit "failed to create backup"
		fi
		# Clean up any socket files that may have been partially copied
		find "$BACKUP_DIR/infra" -type s -exec rm -f {} \; 2>/dev/null
		log "backup created at $BACKUP_DIR"
	else
		log "no existing installation found to backup"
	fi
	log "backup ok"
}

# SERVER-1931: rollback_whatap no longer starts the service itself. The
# caller writes the patch result first (after the restore, because the
# restore wipes $INSTALL_DIR and would delete a record written earlier) and
# only then calls start_service, so the agent never comes up before the
# final result is recorded.
rollback_whatap() {
	log "rollback start..."
	stop_service
	if [ -n "$INSTALL_DIR" ] && [ "$INSTALL_DIR" != "/" ]; then
		rm -rf "$INSTALL_DIR" 2>/dev/null
	fi
	mkdir -p "$INSTALL_DIR"
	(cd "$BACKUP_DIR/infra" && cp -Rp . "$INSTALL_DIR/") || error_exit "failed to rollback"
	log "rollback ok"
}

install_package() {
	log "install $pkg ..."

	cd /tmp
	gunzip -c "$pkg" | tar xvf -
	if [ $? -ne 0 ]; then
		error "$pkg extract fail"
		PATCH_FAIL_MSG="Package extract failed"
		return 1
	fi

	if [ -x "./whatap-infra/install.sh" ]; then
		# install.sh defaults INSTALL_ROOT to /usr when no path arg is given,
		# which reinstalls to /usr/whatap/infra and breaks updates of agents
		# installed to a custom path. Derive the original install root from
		# INSTALL_DIR (e.g. /opt/whatap/infra -> /opt) and pass it through.
		INSTALL_ROOT=$(echo "$INSTALL_DIR" | sed 's:/*$::; s:/whatap/infra$::')
		[ -z "$INSTALL_ROOT" ] && INSTALL_ROOT=/usr
		log "reinstalling to INSTALL_ROOT=$INSTALL_ROOT (from INSTALL_DIR=$INSTALL_DIR)"
		awk 'BEGIN{for(i=0;i<20;i++)print "n"}' | ./whatap-infra/install.sh "" "" "$INSTALL_ROOT"
		rc=$?
		cd /
		if [ $rc -ne 0 ]; then
			error "$pkg install fail"
			PATCH_FAIL_MSG="Package install script failed"
			rm -rf "whatap-infra"
			return 1
		fi
	else
		error "install.sh not found in package"
		PATCH_FAIL_MSG="install.sh not found in package"
		rm -rf "whatap-infra"
		return 1
	fi

	rm -rf "whatap-infra"
	log "$pkg install ok"
	return 0
}

test_agent() {
	log "testing agent..."

	# SERVER-1931: each failure point sets a distinct PATCH_FAIL_MSG so
	# patch_history.log identifies where the test failed. The first two
	# failures happen before boot-test even runs, so a generic message made
	# whatap_infra_boottest.log (still holding the previous success) look
	# contradictory and delayed root-cause analysis.
	if [ -x "$INSTALL_DIR/whatap_infrad" ]; then
		version_output=$("$INSTALL_DIR/whatap_infrad" version 2>&1)
		if [ $? -ne 0 ]; then
			error "Agent Version Check Failed"
			PATCH_FAIL_MSG="Agent version check failed (boot-test not executed)"
			return 1
		fi
		echo "$version_output"
		PATCH_VERSION=$(echo "$version_output" | head -1 | awk '{print $NF}')
		export PATCH_VERSION
		export WHATAP_HOME="$INSTALL_DIR/conf"
		"$INSTALL_DIR/whatap_infrad" boot-test
		if [ $? -ne 0 ]; then
			error "Agent Test Check Failed"
			PATCH_FAIL_MSG="Agent boot-test failed"
			return 1
		fi
		return 0
	else
		error "agent test fail"
		PATCH_FAIL_MSG="Agent binary missing or not executable (boot-test not executed)"
		return 1
	fi
}

check_root() {
	log "checking root privileges..."
	if [ "$(id -u)" -ne 0 ]; then
		error_exit "This script must be run as root"
	fi
	log "check root ok"
}

# --- Main ---
check_root

PATCH_START_TIME=$(date '+%Y-%m-%d %H:%M:%S')
PATCH_VERSION=''
PATCH_FAIL_MSG=''
log "Starting patch process : $PATCH_START_TIME"

write_patch_inprogress

stop_service
backup_whatap
backup_config

install_package

# SERVER-1931: write_patch_result is called before start_service in every
# path. The outcome is already final at that point, and recording it first
# closes the window in which a restarted agent would answer a status query
# with the previous patch result. On the rollback path the result is written
# after rollback_whatap (the restore wipes $INSTALL_DIR, so an earlier write
# would be lost) but still before start_service; the in-progress marker
# covers the interval until then. start_service failures were never
# reflected in patch_history before either (log only), so no behavior is
# lost.
if [ $? -ne 0 ]; then
	[ -z "$PATCH_FAIL_MSG" ] && PATCH_FAIL_MSG="Package installation failed"
	log "Package installation failed - rollback"
	rollback_whatap
	write_patch_result "rollback" "$PATCH_VERSION" "$PATCH_START_TIME" "$PATCH_FAIL_MSG"
	start_service
else
	log "install complete"
	restore_config
	test_agent
	if [ $? -ne 0 ]; then
		[ -z "$PATCH_FAIL_MSG" ] && PATCH_FAIL_MSG="Agent test failed"
		log "Agent test failed - rollback"
		rollback_whatap
		write_patch_result "rollback" "$PATCH_VERSION" "$PATCH_START_TIME" "$PATCH_FAIL_MSG"
		start_service
	else
		log "Patch completed successfully"
		write_patch_result "success" "$PATCH_VERSION" "$PATCH_START_TIME" ""
		start_service
	fi
fi
