#!/bin/bash
###############################################################################
# The idea behind this script is to take a load of input files and see if the #
# potental output files pass a series of tests. If they don't then they form  #
# the input for a cluster job array, if they do then they are not run         #
###############################################################################

SCRIPT_LOCATION="$( readlink -f "${BASH_SOURCE[${#BASH_SOURCE[@]} - 1]}" )"
PROG_DIR="$( dirname "$SCRIPT_LOCATION")"
PROG_NAME="$(basename "$SCRIPT_LOCATION")"
VERSION="0.1"

################################################################################
############################ FUNCTIONS ARE HERE ################################
################################################################################


# Function Description
################################################################################
#
################################################################################
error() {
    local lineno=$1
    local error="${2:-error!}"
    
    FLAGS_verbose=0
    msg "$error" "error" "true"

    delete_temp_files
    exit 1
}


# Function Description
################################################################################
#
################################################################################
delete_temp_files() {
    for i in "${TEMP[@]}"; do
        msg "deleting '$i'..." "cleanup"
        rm "$i"
    done
}


# Function Description
################################################################################
#
################################################################################
msg() {
    local string="$1"
    local type="${2:-info}"
    local stderr="${3:-false}"
    local level="${4:-1}"

    string="[$type] $string"

    if [ $FLAGS_verbose -eq 0 ] && [ $level -eq 1 ]; then
        case "$stderr" in
            "false")
                echo "$string"
                ;;
            "true")
                echo "$string" 1>&2
                ;;
            *)
                error $LINENO "unknown STDERR option: '$stderr'"
                ;;
        esac
    elif [ $FLAGS_xverbose -eq 0 ] && [ $level -eq 2 ]; then
        case "$stderr" in
            "false")
                echo "$string"
                ;;
            "true")
                echo "$string" 1>&2
                ;;
            *)
                error $LINENO "unknown STDERR option: '$stderr'"
                ;;
        esac        
    fi
    
}


# get_files
################################################################################
# This will supply a file containing all the files to check. This default      #
# version. Will write INFILES to a file (if they are supplied). If they are    #
# not supplied it will check for a jafile.                                     #
################################################################################
get_files() {
    # First see if any input files have been passed, if they have then
    # we use them and ignore any --jafiles. Note that we do not check 
    # --get_files as if it is present this will not be called! 
    if [ ${#INFILES[@]} -gt 0 ]; then
        # If there are input files then we will write them to disk
        local files=$(mktemp -p "$FLAGS_temp")
        TEMP+=("$files")

        for i in "${INFILES[@]}"; do
            echo "$i" >> "$files"
        done

        # "Return" the file name
        MASTER_FILE="$files"
    elif [ ! -z $FLAGS_jafile ]; then
        # if a job array file has been given "return" that
        MASTER_FILE="$FLAGS_jafile"
    else
        # if we do not have any of these things then it is an error
        error $LINENO "no input files supply input files, --jafile or get_files()"

        # Not sure why this is not working in the error function above
        exit 1
    fi

}


# file_exists
################################################################################
# Tests if a file exists and is readable                                       #
################################################################################
file_exists() {
    local test_file="$1"

    # If the test file exists and is readable then we pass the test
    if [ -e "$test_file" ] && [ -r "$test_file" ]; then
        echo "pass"
    else
        # otherwise we fail
        echo "fail"
    fi
}


# file_size_ge
################################################################################
# Tests if a file is greater than or = to a certain size in kb. If the file    #
# does not exist then this is a fail                                           #
################################################################################
file_size_ge() {
    local test_file="$1"
    local size=$2
    local exists=$3
    # echo $exists
    if [ -z $exists ]; then
        exists=$(file_exists "$test_file")
    fi
    # echo "EXISTS="$exists 1>&2
    if [ $exists == $PASS ]; then
        local file_size=$(du -k "$test_file" | cut -f1)
        # echo "FILE SIZE="$file_size 1>&2
        # echo "TEST SIZE="$size 1>&2

        if [[ $file_size -ge $size ]]; then
            # echo "PASSING" 1>&2
            echo $PASS
        else
            # echo "FAILING" 1>&2
            echo $FAIL
        fi
    else
        echo $FAIL
    fi
}


# do_tests
################################################################################
# perform all the file tests required by the user, failure of 1 is a total     #
# failure                                                                      #
################################################################################
do_tests() {
    # The name of the file that will be tested
    local test_file="$1"

    # The tests that will be performed 0=true, 1=false
    local exists=$2
    local size_ge=$3

    # echo "local exists $test_file="$exists
    # if we are checking if inputfiles exist
    if [[ $exists -eq 0 ]]; then
        # msg "testing exists ($exists): $test_file" false 2
        test_report["exists"]=$(file_exists "$test_file")
        # msg "testing exists ($exists): ${test_report[exists]}" false 2
    fi

    # if we are checking if the file >= size test needs doing 
    if [[ $size_ge -ge 0 ]]; then
        # msg "testing size >= ($size_ge): $test_file " false 2
        test_report["exists"]=$(file_exists "$test_file")
        test_report["size_ge"]=$(file_size_ge "$test_file" "$size_ge" ${test_report["exists"]})
        # msg "testing size >= ($size_ge): ${test_report[size_ge]}" false 2
    fi

}


# evaluate
################################################################################
# This evaluates all the results from the tests, generates output and          # 
# summary counts                                                               #
# test_type: input or output                                                   #
# line_no: The record number that is being processed                           #
# test_file: The file that is being processed                                  #                
################################################################################
evaluate() {
    local test_type="$1"
    local line_no="$2"
    local test_file=$(basename "$3")

    # msg "line no: $line_no; test_file: $test_file" "evaluate|$test_type" false 2

    # # overall is a global variable that is initialised to pass, this will be
    # # changed depending on the results of the tests
    # overall="pass"

    # A summary string will be built up for each record, this will only be 
    # output if --verbose
    local string="$line_no: $test_file"
    
    # We use this to monitor situations where there have been no tests
    local ntests=0

    # The fields are the names of tests that potentially have been conducted on 
    # the file
    # Loop through all the fields, not all of these will have been tested
    # so we only log those that have
    for i in "${fields[@]}"; do
        # msg "looking for field: $i" "evaluate|$test_type" false 2
        # see if the global test report is empty for that field
        if [ ! -z ${test_report[$i]} ]; then
            # msg "found field: $i; value: ${test_report[$i]}" "evaluate|$test_type" false

            # increment the number of tests
            ntests=$((ntests + 1))

            # build the string up
            string=$string", ${i}=${test_report[$i]}"

            # If the test has failed
            if [ "${test_report[$i]}" == "fail" ]; then
                # if one test fails then everything fails
                overall="fail"

                # Increment the failures for this test
                TOTALS[${test_type}"_"${i}]=$((TOTALS[${test_type}"_"${i}] + 1))
            fi
        fi
    done

    # If there have not been any tests then we log that although it is 
    # logically counted as a pass
    if [[ $ntests -eq 0 ]]; then
        overall="untested"
    fi

    # Update the overall score
    TOTALS[${test_type}"_"${overall}]=$((TOTALS[${test_type}"_"${overall}] + 1))

    # output the msg should we be verbose
    msg "$string" "$test_type|$overall" "false" 2
}


################################################################################
############################ FUNCTIONS END HERE ################################
################################################################################


# The usage instructions, these will be output if the user gets anything wrong
USAGE=`cat << EOF
${PROG_NAME} v$VERSION
usage: ${PROG_NAME} options [infiles]

[infiles]  One or more input files that will be checked (optional). If these are not defined then either '--jafile' should be given or '--custom' get_files function should be passed.

The idea behind this script is to take a load of input files and see if the potental output files pass a series of tests. If they don't then they form the input for a cluster job array, if they do then they are not run.

problems: c.finan@ucl.ac.uk

EOF
`

# Don't set trap errors before using this otherwise it doesn't work properly
# this sources in shflags. This is a third party code for streamlining bash 
# arguments and seems to work well. This should be in your path.
. shflags || echo $USAGE

# Default argument values here
DEFINE_string 'jafile' '' 'a file with job array parameters, input file is first column, output file is second column' 't'
DEFINE_string 'outpath' '' 'if provided an output file with the basename as the input will be tested in this path' 'p'
DEFINE_string 'runfile' "${PWD}/ja_run.txt" 'the job array run file for the output file files that have failed' 'r'
DEFINE_integer 'osize_ge' -1 'test if the output file size is >= --osize_ge kb (< 0 = off)'
DEFINE_integer 'isize_ge' -1 'test if the input file size is >= --isize_ge kb (< 0 = off)'
DEFINE_boolean 'outexist' false 'test if the output files exist' 'o'
DEFINE_boolean 'inexist' true 'test if the input files exist' 'i'
DEFINE_boolean 'verbose' false 'do you want to print some info' 'v'
DEFINE_boolean 'xverbose' false 'do you want to print loads of info' 'x'

FLAGS_HELP=${USAGE}

# parse the command-line
FLAGS "$@" || exit 1
eval set -- "${FLAGS_ARGV}"

trap "error $LINENO" ERR SIGINT
trap "delete_temp_files" EXIT

# This will hold any temp files that are used by this script
TEMP=()

# These arrays are used to loop through associative arrays
# in a defined order
fields=('exists' 'size_ge')
types=('input' 'output')
summary_fields=('pass' 'untested' 'fail')

# Will hold summary counts of failures to display to the user
declare -A TOTALS=()

PASS="pass"
FAIL="fail"
UNTESTED="untested"

# Grab the positional arguments
INFILES=($@)

# Get the full path of the runfile that will be created
FLAGS_runfile=$(readlink -f "$FLAGS_runfile")

msg "** $PROG_NAME v$VERSION **"
msg "true=0, false=1"
msg "verbose: $FLAGS_verbose"
msg "outexist: $FLAGS_outexist"
msg "runfile: $FLAGS_runfile"
msg "inexist: $FLAGS_inexist"
msg "output size >=: $FLAGS_osize_ge"
msg "input size >=: $FLAGS_isize_ge"
msg "jafile: $FLAGS_jafile"
msg "outpath: $FLAGS_outpath"
msg "checking input file source..."

# This function sets a variable called MASTER_FILE that points to the location
# of a file with al the files to check
get_files

msg "input file in: $MASTER_FILE"

# Make sure the run file is empty before we append to it
rm -f "$FLAGS_runfile"

# If we get here we should have some input files to play around with, so we now go 
# through the files and perform tests according to the options
line=1 # for logging the line number of the input

# Read in the input the first and second lines we expect to be
# the input/output files, the rest we do not care. The input
# should be tab delimited
while IFS=$'\t' read -r "infile" "outfile" "remainder"; do
    # This is declared/redeclared on every call
    # as it is global
    unset $test_report
    declare -A test_report=()

    # initialise the overall variable to pass, this is a global variable
    # that is set when evaluate is called
    overall="$PASS"

    # this is used in case $output is redefined based on outpath being given,
    # this is where the user wants to generate output file names based on
    # outpath and input file names
    outfile_bak=
    
    # if we are extra verbose 
    msg "line $line: $infile" "infile" false 2
    
    # Do any tests that are required on the input files
    do_tests "$infile" $FLAGS_inexist $FLAGS_isize_ge
    evaluate "input" "$line" "$infile"
    infile_overall="$overall"
    # echo $infile_overall
    # Now test the output files (if required)
    # if the user has defined an outpath then we append the input file basename
    # to the outpath and use that location for the tests
    if [ ! -z $FLAGS_outpath ]; then
        # in this case it is likely that any variables in the outfile
        # variable are not outfiles but something else the user requires
        # in which case we want to back them up for outputting later
        outfile_bak="$outfile"
        outfile="$FLAGS_outpath"/$(basename "$infile")
    fi
    # echo ${test_report[@]}
    # This is declared/redeclared on every call
    # as it is global
    unset $test_report
    declare -A test_report=()
    overall="$PASS"

    # we may not be testing the output file so we initialise it to fail
    # that way all the jobs will be run based on PASS/FAIL of the input file
    outfile_overall="$FAIL"

    # we can only test output files if they are defined
    if [ ! -z $outfile ]; then
        # echo "OUT_EXIST="$FLAGS_outexist
        do_tests "$outfile" $FLAGS_outexist $FLAGS_osize_ge
        evaluate "output" "$line" "$outfile"

        outfile_overall="$overall"
    fi

    # Now we evaluate what needs outputting based on the overall input file
    # PASS/FAIL and the overall output file PASS/FAIL
    # the rules are quite simple, if the input file fails then the row is not 
    # included in the run file. if the input file passes and the output file
    # fails it is included in the job array. If the input file passes and the
    # ouptput file passes then it is NOT included in the job array
    if [ "$infile_overall" == "$PASS" ] || [ "$infile_overall" == "$UNTESTED" ]; then
        # Now test the output file overall variable to see if the row
        # needs outputting to job_array_file
        if [ "$outfile_overall" == "$FAIL" ] || [ "$outfile_overall" == "$UNTESTED" ]; then
            cols=""
            # loop through the columns and output tab delimited
            for i in "${outfile}" "${outfile_bak}" "${remainder}"; do
                if [ ! -z "$i" ]; then
                    cols="$(echo -e "${cols}\t$i")"
                fi
            done
            echo -e "${infile}${cols}" >> "$FLAGS_runfile"
        fi
    fi

    line=$((line + 1))
done < "$MASTER_FILE"

for j in "${types[@]}"; do
    echo -e "\nsummary $j (# fails)\n=============="
    for i in "${fields[@]}"; do
        value=${TOTALS[${j}_${i}]:-0}
        printf "%-18s%s\n" "${i}_${j}:" "$value"
        # echo -e "${i}_${j}: $value"
    done
done

echo -e "\noverall summary\n==============="
printf "%-18s%s\n" "#files:" "$((line - 1))"
for i in "${types[@]}"; do
    for j in "${summary_fields[@]}"; do
        value=${TOTALS[${i}_${j}]:-0}
        printf "%-18s%s\n" "${i}_${j}:" "$value"
        # echo -e "${i}_${j}: $value"
    done
done

msg "** END **"
