#!/bin/bash
 
# Change into the ~/eeTeam directory
if [ ! -d ~/eeTeam ]
then
	mkdir ~/eeTeam
fi
cd ~/eeTeam

# Define variables
mm_version="1.1a"			# Memory (and CPU Usage, and Threads, and FDs) Monitor Version
monitor_delay=10			# Monitor delay
next_monitor=0				# Next monitor time
script_id=$$				# Script ID
quiet_mode=0				# Quiet mode, show only important information
my_date=$(date +"%D, %T")	# Current date and time
mm_log="sm_log.csv"			# Memory Monitor Log File
top_buf=""					# Holds top information
proc_str=""					# Grepped process information
log_data=0					# Log Data

# Temperature variables
declare -i temp_curr		# Current Temperature
declare -i temp_limit		# Temperature Limit
declare -i temp_trip_time	# Time Temperature Tripped Over Limit
declare -i temp_watch		# Watch For Temperature Over Limit
temp_curr=0
temp_limit=100000
temp_trip_time=0
temp_watch=1
trans_curr=0

declare -i iDmy

# Get System specific information
inet_address=$(hostname -I | cut -f1 -d' ')
curr_directory=$(pwd)

# List of processes
#declare -a procList=('project.exe' 'kswapd0' 'chromium-browse')
declare -a procList=('project.exe' 'chromium-browse')


# ########################################
# For chromium rolling average CPU usage
#
chCpuProc="chromium-browse"	# Process name (to watch for)
#declare -i chCpuArray		# CPU usage array
declare -i chCpuIndex		# Array index
declare -i rollPoints		# Number of points for rolling average
#declare -i rollCpuAvg		# Rolling average
#declare -i chCpuCurrent	# Current CPU usage (in 100ths)

# Kill Chromium Variables
declare -i killChromium		# Kill Chromium
declare -i chCpuKill		# Enable chromium kill on chromium CPU Usage
declare -i chCpuThold		# Maximum CPU usage (in 100ths) before killing chromium
declare -i chCpuCurrent		# Current CPU usage (in 100ths)
declare -i chMemKill		# Enable chromium kill on chromium Memory Usage
declare -i chMemThold		# Maximum Memory usage (in 100ths) before killing chromium
declare -i chMemCurrent		# Current Memory usage (in 100ths)
declare -i chSwapKill		# Enable chromium kill on chromium Swap Memory
declare -i chSwapThold		# Maximum chromium Swap Memory before killing chromium
declare -i chSwapCurrent	# Current chromium Swap Memory
declare -i chOvSwapKill		# Enable chromium kill on Overall Swap Memory
declare -i chOvSwapThold	# Maximum Overall Swap Memory before killing chromium
declare -i chOvSwapCurrent	# Current Overall Swap Memory

# Flush Swap Memory variables
declare -i flushswapmemory	# Flush Swap Memory
declare -i nextflushswapmemory	# Next Swap Memory Flush allowed
declare -i flushonkill		# Swap Memory Flush on chromium kill
declare -i flushonswap		# Swap Memory Flush on swap memory used
declare -i maxflushovswap	# Maximum Swap Memory used before forcing swap memory flush
declare -i flushwindow		# Window of time before swap memory can be flushed again

# Initialize Kill Chromium related variables
killChromium=0
chCpuIndex=0
rollPoints=5
rollCpuAvg=0
chCpuKill=0
chCpuThold=8000
chCpuCurrent=0
chMemKill=0
chMemThold=6000
chMemCurrent=0
chSwapKill=0
chSwapThold=250000
chSwapCurrent=0
chOvSwapKill=0
chOvSwapThold=250000
chOvSwapCurrent=0
chAction=""

# Initialize Flush Swap Memory related variables
flushswapmemory=0
nextflushswapmemory=$SECONDS
flushonkill=0
flushonswap=0
maxflushovswap=20000
flushwindow=30
fsAction=""



for ((i=0 ; i <= $rollPoints ; i++));
do
	chCpuArray[$!i]=0
done
#
# For chromium rolling average CPU usage
# ########################################


# Function Variables (dumb, I know)
f_ProcName=""
f_PidNumber=""
f_CpuUsage=""
f_MemUsage=""
f_SwapMem=""
f_Threads=""
f_FileDesc=""
output_string="output"


# f_getProcHeading
# Gets log file header (column name) information
# for process being monitored
f_getProcHeading ()
{
	# Get Process Name
	for f_ProcName in "${procList[@]}"
	do
		output_string+=", "$f_ProcName"_PID, "$f_ProcName"_cpu, "$f_ProcName"_mem, "$f_ProcName"_swap, "$f_ProcName"_threads, "$f_ProcName"_fdesc"
	done
}


# f_getProcInfo
# Gets CPU Usage, Memory Usage, Swap Memory Usage, 
# Number of Threads, and File Descriptors
# information for monitored processes
# Appends to output string
f_getProcInfo ()
{
	# Get Process Name
	for f_ProcName in "${procList[@]}"
	do
		f_PidNumber=0
		vArray=($(echo "$top_buf" | grep "$f_ProcName" | awk '{print $1}'))
		f_PidNumber="${vArray[0]}"
		for val in "${vArray[@]}"
		do
			if [ "$f_PidNumber" -gt "$val" ]
			then
				f_PidNumber="$val"
			fi
		done

		f_CpuUsage=0
		vArray=($(echo "$top_buf" | grep "$f_ProcName" | awk '{print $9}'))
		for val in "${vArray[@]}"
		do
			f_CpuUsage=$(echo "$f_CpuUsage $val" | awk '{printf "%f" , $1 + $2}')
		done

		f_MemUsage=0
		vArray=($(echo "$top_buf" | grep "$f_ProcName" | awk '{print $10}'))
		for val in "${vArray[@]}"
		do
			f_MemUsage=$(echo "$f_MemUsage $val" | awk '{printf "%f" , $1 + $2}')
		done

		f_SwapMem=0
		for pidnum in $(pidof $f_ProcName)
		do
			val=$(awk '/VmSwap/{print $2}'  /proc/$pidnum/status)
			f_SwapMem=$(echo "$f_SwapMem $val" | awk '{printf "%f" , $1 + $2}')
		done

		f_Threads=0
		for pidnum in $(pidof $f_ProcName)
		do
			val=$(ps huH p $pidnum | wc -l)
			((f_Threads=f_Threads+val))
		done

		f_FileDesc=0
		for pidnum in $(pidof $f_ProcName)
		do
			val=$(sudo ls /proc/$pidnum/fd -l | wc -l)
			((f_FileDesc=f_FileDesc+val))
		done

		# Append data
		output_string+=", $f_PidNumber, $f_CpuUsage, $f_MemUsage, $f_SwapMem, $f_Threads, $f_FileDesc"
		
		# Is this the process we are watching?
		if [ "$f_ProcName" == "$chCpuProc" ]
		then
			#iDmy=$!f_CpuUsage
			chCpuArray[$chCpuIndex]=$(echo "$f_CpuUsage" | awk '{printf "%f" , $1}')
			((chCpuIndex=(chCpuIndex+1)%rollPoints))
			chMemCurrent=$(echo "$f_MemUsage" | awk '{printf "%d" , $1 * 100}')
			chSwapCurrent=$(echo "$f_SwapMem" | awk '{printf "%d" , $1}')
		fi
	done
}


# f_killProcess - Kills a specified process
# Arguments: $1 - Process Name
# Returns:   1  - Process terminated normally (-SIGINT  / -2)
#            2  - Process killed forcefully   (-SIGKILL / -9)
#            10 - No process name specified
#            11 - No process PIDs found
f_killProcess ()
{
	# Was a process name specified (first argument)
	if [ -z $1 ]
	then
		# No process name, get out
		chAction="No Proc Name"
		return 10
	fi

	# Get Process PIDs
	#vArray=($(echo "$top_buf" | grep "$1" | awk '{print $1}'))
	vArray=($(pgrep chromium-browse))
	if [ ${#vArray[@]} -le 0 ]
	then
		# No PIDs found for process
		chAction="No Proc PID"
		return 11
	fi

	# Kill process gracefully, convince it to die
	chAction="Kill SIGINT"
	pkill -SIGINT $1

	# Verify that all the previously detected PIDs are gone...
	for ((loop=0 ; loop <= 5 ; loop++));
	do
		sleep 1
		count=0
		for val in "${vArray[@]}"
		do
			if ps -p $val > /dev/null
			then
				((count++))

				# Something is still alive,
				# no point on checking other PIDs
				if [ $loop -lt 5 ]
				then
					break
				fi
			fi
		done

		# Was any PID still active?
		if [ $count -eq 0 ]
		then
			chAction="Kill SIGINT ($loop)"
			# No, we are done, just get out
			return 0
		fi
	done

	# If I made it this far, there were still some active PIDs
	# Euthanize process...
	chAction="Kill SIGKILL ($count/${#vArray[@]})"
	pkill -SIGKILL $1
	
	return 0
}


#########################
# Get parameters
#
for ((i=1 ; i <= $# ; i++));
do

	case "${!i}" in

		# Kill chromium by chromium CPU USage
		"-kchrome" | "-kchcpu")
			chCpuKill=1
			;;

		"-maxchcpu="*)
			chCpuThold=`expr ${!i#*=}`
			;;


		# Kill chromium by chromium Memory Usage 
		"-kchmem")
			chMemKill=1
			;;

		"-maxchmem="*)
			chMemThold=`expr ${!i#*=}`
			;;


		# Kill chromium by chromium Swap Memory 
		"-kchswap")
			chSwapKill=1
			;;

		"-maxchswap="*)
			chSwapThold=`expr ${!i#*=}`
			;;


		# Kill chromium by Overall Swap Memory 
		"-kchovswap")
			chOvSwapKill=1
			;;

		"-maxovswap="*)
			chOvSwapThold=`expr ${!i#*=}`
			;;


		# Flush Memory Swap on Chromium Kill 
		"-flushonkill")
			flushonkill=1
			;;


		# Flush Swap Memory on System Memory Swap 
		"-flushonswap")
			flushonswap=1
			;;

		"-maxflushovswap="*)
			maxflushovswap=`expr ${!i#*=}`
			;;

		"-flushwindow="*)
			flushwindow=`expr ${!i#*=}`
			;;


		# Log Data
		"-log")
			log_data=1
			;;

		"-nolog")
			log_data=0
			;;


		*)
			monitor_delay=`expr ${!i}`
			#monitor_delay=$(echo ${!i} | awk '{printf "%2f" , $1}')
			;;

	esac

done;


if [ $monitor_delay -le 0 ]
then
	monitor_delay=5
fi

#if [ $run_time -le 15 ]
#then
#	run_time=15
#fi

#
# Get Parameters
#########################


#if [ $quiet_mode -eq 0 ]
#then

	echo ""
	echo "$0 - Super Monitor, v$mm_version"
	echo ""
	echo "System Information"
	echo "    INet Address:      $inet_address"
	echo "    Current Directory: $curr_directory"
	echo "    Process ID:        $$"
	echo ""
	echo "Usage: "$0" [Monitor_Delay] [-kchrome] [-kchcpu] [-maxchcpu=x] [-kchmem] [-maxchmem=x] [-kchOvSwap] [-maxchswap=x] [-log] [-nolog]"
	echo "                  [-flushonkill] [-flushonswap] [-maxflushovswap=x] [-flushwindow=x]"
	echo "    Where:    Monitor_Delay      Delay between Monitoring loops."
	echo "                                 Omit or set to <= 0 for 5 seconds"
	echo "              -kchrome           Enable chromium kill by chromium CPU usage (same as -kchcpu)"
	echo "              -kchcpu            Enable chromium kill by chromium CPU usage"
	echo "              -maxchcpu=x        Set maximum chromium CPU usage (in 100ths) to 'x' before killing chromium"
	echo "              -kchmem            Enable chromium kill by chromium Memory Usage"
	echo "              -maxchmem=x        Set maximum chromium Memory usage (in 100ths) to 'x' before killing chromium"
	echo "              -kchswap           Enable chromium kill by chromium Swap Memory"
	echo "              -maxchswap=x       Set maximum chromium Swap Memory to 'x' before killing chromium"
	echo "              -kchovswap         Enable chromium kill by Overall Swap Memory"
	echo "              -maxovswap=x       Set maximum Overall Swap Memory to 'x' before killing chromium"
	echo "              -flushonkill       Enable Flush Swap Memory on chromium kill"
	echo "              -flushonswap       Enable  Flush Swap Memory on overall used swap memory"
	echo "              -maxflushovswap=x  Set maximum Overall Used Swap Memory before forcing a swap memory flush"
	echo "              -flushwindow=x     Set minimum time before attempting another swap memory flush"
	echo "              -log               Enable data logging"
	echo "              -nolog             Disable data logging"
	echo ""

	echo "Delay between loops:                       "$monitor_delay
	echo "Kill chromium by chromium CPU:             "$chCpuKill
	echo "Maximum chromium CPU % (100ths):           "$chCpuThold
	echo "Kill chromium by chromium Memory:          "$chMemKill
	echo "Maximum chromium Memory (100ths):          "$chMemThold
	echo "Kill chromium by chromium Swap Memory:     "$chSwapKill
	echo "Maximum chromium Swap Memory:              "$chSwapThold
	echo "Kill chromium by Overall Swap Memory:      "$chOvSwapKill
	echo "Maximum Overall Swap Memory:               "$chOvSwapThold
	echo "Flush Swap Memory on Chromium Kill:        "$flushonkill
	echo "Flush Swap Memory on System Swap Memory:   "$flushonswap
	echo "Maximum System Swap Memory before flush:   "$maxflushovswap
	echo "Minimum time between Swap Memory flushes:  "$flushwindow
	echo ""
	echo ""

	if [ $log_data -ne 0 ]
	then
		echo "Log file will be generated"
	fi

#	echo ""

#fi

#sleep 1


#########################
# Log files information
#
f_curr=""       # Current file
f_new=""        # Newest file
f_old=""        # Oldest file
f_none=""       # Non-existent file
f_bkp=""	# File name to backup previous accumulated file to...
f_found=0       # Existing file has been found

if [ $log_data -ne 0 ]
then

	for fnum in {0..99}
	do
		# What is the current file name?
		f_curr="sm_"$fnum".csv"

		# Does file exist?
		if [ ! -e "$f_curr" ]
		then
		        # First non-existing file
		        if [ ${#f_none} -eq 0 ]
		        then
		                f_none="$f_curr"
		        fi

		        continue
		fi

		# File exists...  If this is the first one,
		# assign to newest and oldest and continue.
		if [ $f_found -eq 0 ]
		then
		        f_new="$f_curr"
		        f_old="$f_curr"
		        f_found=1
		        continue
		fi

		# Check for Newest File
		if [ "$f_curr" -nt "$f_new" ]
		then
		        f_new="$f_curr"
		fi

		# Check for Oldest File
		if [ "$f_curr" -ot "$f_old" ]
		then
		        f_old="$f_curr"
		fi

	done

	if [ -e "$mm_log" ]
	then
		# Copy to Non-Existent or Oldest?
		if [ ${#f_none} -ne 0 ]
		then
			f_bkp="$f_none"
		else
			f_bkp="$f_old"
		fi
	fi

	echo "Non existing file: $f_none"
	echo "Newest file:       $f_new"
	echo "Oldest file:       $f_old"
	echo "Backup file:       $f_bkp"
	echo ""

	#sleep 3

	if [ ${#f_bkp} -ne 0 ]
	then
		echo "Current $mm_log file to be backed up to $f_bkp"
		mv -f $mm_log $f_bkp
	fi

fi

#
# Log files information
#########################


# Initialize log file
if [ $log_data -ne 0 ]
then

	echo "$0 - Super Monitor, v$mm_version" > $mm_log
	echo "$my_date" >> $mm_log
	if [ ${#f_bkp} -ne 0 ]
	then
		echo "Current $mm_log file to be backed up to $f_bkp" >> $mm_log
	fi

	# System specific information
	echo "" >> $mm_log
	echo "System Information" >> $mm_log
	echo "    INet Address:      $inet_address" >> $mm_log
	echo "    Current Directory: $curr_directory" >> $mm_log
	echo "    Process ID:        $$" >> $mm_log
	echo "" >> $mm_log

	echo "Delay between loops:                       "$monitor_delay >> $mm_log
	echo "Kill chromium by chromium CPU:             "$chCpuKill >> $mm_log
	echo "Maximum chromium CPU % (100ths):           "$chCpuThold >> $mm_log
	echo "Kill chromium by chromium Memory:          "$chMemKill >> $mm_log
	echo "Maximum chromium Memory (100ths):          "$chMemThold >> $mm_log
	echo "Kill chromium by chromium Swap Memory:     "$chSwapKill >> $mm_log
	echo "Maximum chromium Swap Memory:              "$chSwapThold >> $mm_log
	echo "Kill chromium by Overall Swap Memory:      "$chOvSwapKill >> $mm_log
	echo "Maximum Overall Swap Memory:               "$chOvSwapThold >> $mm_log
	echo "Flush Swap Memory on Chromium Kill:        "$flushonkill >> $mm_log
	echo "Flush Swap Memory on System Swap Memory:   "$flushonswap >> $mm_log
	echo "Maximum System Swap Memory before flush:   "$maxflushovswap >> $mm_log
	echo "Minimum time between Swap Memory flushes:  "$flushwindow >> $mm_log

	echo "" >> $mm_log
	echo "" >> $mm_log

fi

##############################
# Add data header to csv file
#

output_string="SID, Date, Time, SOM_Temp, CPU_Trans, Sys_Mem, Swap_Mem"

# Get headings for monitored processes
f_getProcHeading

# Add rolling average header
output_string+=", cpu_roll_avg"

# and Chromium and Flush Swap Action
output_string+=", ch_action, fs_action"

# Append header to output file
if [ $log_data -ne 0 ]
then
	echo $output_string | tee -a $mm_log
else
	echo $output_string
fi

#
# Add data header to csv file
##############################


##############################
# Test loop
#

next_monitor=$(($SECONDS+$monitor_delay))
while true
do

	# Default chromium and flush memory action to idle
	chAction="Idle"
	fsAction="Idle"

	# Get current Date and Time (for Time Stamp)
	my_date=$(date +"%D, %T")

	# Start defining output string
	output_string="$script_id, $my_date"

	# SOM Temperature
	temp_curr=`cat /sys/bus/platform/devices/temp_sensor_hwmon.0/temp1_input`
	output_string+=", $temp_curr"

	# CPU Transitions
	trans_curr=`cat /sys/devices/system/cpu/cpu1/cpufreq/stats/total_trans`
	output_string+=", $trans_curr"

	# Get top information
	top_buf=$(top -bn 1)

	# Get System Memory Information
	str=($(echo "$top_buf" | grep "Mem:"))
	output_string+=", ${str[3]//k}"

	# Get Swap Memory Information
	str=($(echo "$top_buf" | grep "Swap:"))
	chOvSwapCurrent="${str[3]//k}"
	output_string+=", $chOvSwapCurrent"

	# Get processes information
	f_getProcInfo

	# Calculate cpu rolling average
	((rollCpuAvg=0))
	for ((i=0 ; i < $rollPoints ; i++));
	do
		rollCpuAvg=$(echo "${chCpuArray[$i]} $rollCpuAvg" | awk '{printf "%f" , $1 + $2}')
	done

	rollCpuAvg=$(echo "$rollCpuAvg $rollPoints" | awk '{printf "%f" , $1 / $2}')
	chCpuCurrent=$(echo "$rollCpuAvg" | awk '{printf "%d" , $1 * 100}')
	output_string+=", $rollCpuAvg"


	# Reset the Kill Chromium flag
	killChromium=0
	flushswapmemory=0

	# Kill chromium on CPU usage?
	if [ $chCpuKill != 0 ] && [ $chCpuCurrent -gt $chCpuThold ]
	then
		killChromium=1
	fi

	# Kill chromium on Memory usage?
	if [ $chMemKill != 0 ] && [ $chMemCurrent -gt $chMemThold ]
	then
		killChromium=1
	fi

	# Kill chromium on chromium Swap Memory?
	if [ $chSwapKill != 0 ] && [ $chSwapCurrent -gt $chSwapThold ]
	then
		killChromium=1
	fi

	# Kill chromium on Overall Swap Memory?
	if [ $chOvSwapKill != 0 ] && [ $chOvSwapCurrent -gt $chOvSwapThold ]
	then
		killChromium=1
	fi

	# Flush Swap Memory on Overall Swap Memory?
	if [ $SECONDS -gt $nextflushswapmemory ] && [ $flushonswap != 0 ] && [ $chOvSwapCurrent -gt $maxflushovswap ]
	then
		flushswapmemory=1
	fi



	# Do I need to kill chromium?
	if [ $killChromium != 0 ]
	then
		chAction="Kill"
		#pkill -SIGINT $chCpuProc

		# Need to kill Chromium
		f_killProcess chromium-browse
		
		# Do I need to flush swap memory?
		if [ $flushonkill != 0 ]
		then
			flushswapmemory=1
		fi
		
		# Reset CPU Usage info
		for ((i=0 ; i <= $rollPoints ; i++));
		do
			chCpuArray[$i]=0
		done

	fi


	# Do I need to flush swap memory?
	if [ $flushswapmemory != 0 ]
	then
		fsAction="Flush"
		sudo swapoff -a
		sudo swapon -a
		nextflushswapmemory=$(($SECONDS+$flushwindow))
	fi


	# Add chromium and swap memory flush action
	output_string+=", $chAction, $fsAction"	

	# Append results to output file
	if [ $log_data -ne 0 ]
	then
		echo $output_string | tee -a $mm_log
	else
		echo $output_string
	fi

	# Adjust next_monitor time
	while [ $SECONDS -ge $next_monitor ]
	do
		next_monitor=$(($next_monitor+$monitor_delay))
	done

	# Now, wait to repeat
	delaySecs=$(($next_monitor-$SECONDS))
	if [ $delaySecs -gt 0 ]
	then
		sleep $delaySecs
	fi
done

#
# Test loop
##############################

