#!/bin/bash
# basic attempt at profiling an array job
JOBID="$1"
poll="${2:-15}"
ALLOWED_OVERRUN="${2:-3}"
finalise() {
    echo ""
    if [[ ${#ALL_DATA[@]} -gt 0 ]]; then
        echo "Max Memory:"
        echo ${ALL_DATA[@]} | tr ' ' '\n' | sort -uhr | head -n1
    else
        echo "No Measures:"
        echo "0M"
    fi
    exit 0
}

if [ -z $JOBID ]; then
    echo "no job id!" 1>&2
fi

trap 'finalise' SIGINT ERR

ALL_DATA=()
data=()
start=0
time=0
#poll=2
#ALLOWED_OVERRUN=3
overrun=0
idx=0

# Do a test qstat to determine if the job exists
qstat -j $JOBID >/dev/null 2>&1
if [[ $? -eq 1 ]]; then
    echo "qstat error is the job id correct?" 1>&2
fi

#files=(test_out1.txt test_out1.txt test_out2.txt test_out3.txt test_out3.txt test_out1.txt test_out1.txt test_out1.txt test_out1.txt)
while [[ $start -eq 0 ]]; do
    echo -ne "profiling job: $JOBID.....${time}s [${#ALL_DATA[@]} measures] ${files[$idx]} [CTRL-C] to exit]...\r" 1>&2
    temp_file=$(mktemp)
    qstat -j $JOBID > "$temp_file" 2> /dev/null
    data=($(cat "$temp_file" | perl -ne '$_ =~ m/usage\s+\d+:.+maxvmem=(\d+(\.?\d+)?[MGK])/; if ($1) {print $1."\n"};' | sort -uh))
    ALL_DATA+=(${data[@]})
    sleep $poll
    time=$((time + poll))
    rm "$temp_file"
    # if we are not collecting any more data
    # not the best as will exit if there is a delay in jobs i.e. one finishes before aother starts
    if [[ ${#ALL_DATA[@]} -gt 0 ]] && [[ ${#data[@]} -eq 0 ]]; then
        if [[ $overrun -eq $ALLOWED_OVERRUN ]]; then
            break
        fi
        overrun=$((overrun + 1))
    fi
    idx=$((idx + 1))
done

finalise

