#!/bin/bash
################################ TEMPLATE #####################################
###############################################################################
# A task template to run a command on a subset of rows from a file. The job   #
# array file is expected to have the foloowing columns:                       #
# 0: rowidx                                                                   #
# 1: infile path                                                              #
# 2: outfile path (for a specific chunk)                                      #
# 3: startrow in the file 0-based with the 0 row being a header. This is      #
#     inclusive, a start row of 2 will start at row 2                         # 
# 4: end row. This is exclusive an end row of 3 will not include the row 3 but#
#    will include row 2                                                       #
#                                                                             #
# The job array can be created with ../job_array/file-split-job-array.sh      #
#                                                                             #
# run_command() contains task specific code that will be run for each row in  #
# the job array file. Place your custom code in there.                        #
#                                                                             #
# . task_init.sh should then be called. This initialises everything and       #
# generates a load of global variables that you can use during execution of   #
# the task                                                                    #
#                                                                             #
# The task setup will also deal with the command line input and cleanup       #
# functions                                                                   #
# Globals:                                                                    #
#                                                                             #
# JA="$1"                                                                     #
# TMP_LOC="$2"                                                                #
# STEP=$3                                                                     #
# IDX=${4:-$SGE_TASK_ID}                                                      #
#                                                                             #
# WORKING_DIR : A location for temp files                                     #
#                                                                             #
# Store all the paths to the intermediate files, these will all be deleted    #
# when exitting cleanly or with an error. The "" is to stop unbound errors    #
# TEMPFILES=("")                                                              #
#                                                                             #
# Files that should be moved should go into these two arrays. This will then  #
# be moved to their final location on exit.                                   #
# MOVE_FROM=()                                                                #
# MOVE_TO=()                                                                  #
#                                                                             #
# PROCESS_LINE : An array extracted from the job array file, this provides    #
# input for the current STEP of the starting from the current TASK            #
#                                                                             #
# ROW_IDX : Element 0 from the process line, this is a requirement that all   #
# job arrays have this                                                        #
#                                                                             #
# You will need to have Bash Helpers and shflags in your path for this to work#
# https://gitlab.com/cfinan/bash-helpers                                      #
# https://github.com/kward/shflags                                            #
###############################################################################


# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Test if a file is uncompressed,gzip or bzip2 compressed open and stream to
#  STDOUT.
#
# Globals:
#   None
# Arguments:
#   infile, the path to the input file to test.
# Outputs:
#   File contents streamed to STDOUT
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
test_file() {
	local infile="$1"
	if file -b "$infile" | grep -q "gzip compressed"; then
	    zcat "$infile"
	elif file -b "$infile" | grep -q "bzip2 compressed"; then
		bzip2 -dc "$infile"
	elif file -b "$infile" | grep -q "compressed"; then
	    echo "Can only handle gzip/bzip2 compression" 1>&2
	    exit 1
	else
		cat "$infile"
	fi
}



# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Output rows >= to a start row and < end row to an output file. The header
# row is also output
#
# Globals:
#   None
# Arguments:
#   infile: the path to the input file to test.
#   outfile: the path to the subset output file.
#   start: the start row of the output. 0-based with a header, so first data
#          row is 1
#   end: the end row to stop at.
# Outputs:
#   File contents streamed to STDOUT
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
output() {
	local infile="$1"
	local outfile="$2"
	local start=$3
	local end=$4
	test_file "$infile" | body awk -vstart="$start" -vend="$end" '{
	if (NR>=start && NR<end) {
		print $0
	} else if (NR>=end) {
		exit
	}
	}' | gzip > "$outfile"
}



# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
# Run a single task in the job array file.
#
# Globals:
#   ${PROCESS_LINE[@]}: The line read in from the job array
#   $WORKING_DIR: A tmp directory created to store job data (and deleted after)
# Arguments:
#   None
# Outputs:
#   None
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
run_command() {
    # "${PROCESS_LINE[0]}" is reserved for the row IDX and is not needed for
    # execution

    # Decompose the line array, this is really for code clarity
    infile="${PROCESS_LINE[1]}"
    outfile="${PROCESS_LINE[2]}"
    startrow="${PROCESS_LINE[3]}"
    endrow="${PROCESS_LINE[4]}"

	info_msg "infile: $infile"
	info_msg "outfile: $outfile"
	info_msg "startrow: $startrow"
	info_msg "endrow: $endrow"

	tempin="$(mktemp -p"$WORKING_DIR")"
	tempout="$(mktemp -p"$WORKING_DIR")"
    info_msg "extracting rows (${startrow}, ${endrow}) from: $infile"

	# Output the row subsets that we require
    unset_run_env
	output "$infile" "$tempin" $startrow $endrow
    set_run_env

	# RUN COMMAND HERE AND OUTPUT TO "$tempout"

	# Move tmpout to final
    info_msg "moving finished file to: ${outfile}"
	mv "$tempout" "$outfile"
	
    # Remove the temp row subsets
    rm "$tempin"
    info_msg "finished processing task"
}

. task_init.sh
