#!/bin/bash
######################################################################
#                     qsub (w/ mpirun) on GENERIC cluster.
#-------------------------------------------------------------------
######################################################################

function UJS_Submit
{
	UJS_GetOptions

	if [ $walltime == "unlimited" ]; then
		walltime=01:00:00
	fi
	
	if [ $((npe%nppn)) -ne 0 ]; then
		echo "NOTE: npe=$npe is not dividable by nppn=$nppn. rounding up"
	fi
	

	#omplaceOption="omplace -c 0-127:st=4"
	omplaceOption=""

	echo "Cluster: GENERIC. Scheduler: qsub"
	if [ $nppn -gt $nppnmaxSingleThread ]; then
		qsubrun="mpirun -np $npe $executable $args"
		#qsub doesn't know about hyperthreading. Only specify used procs. Round up.
		# nppnRaw=$(((nppn+1)/2))
		
	else
		qsubrun="mpirun -np $npe $omplaceOption $executable $args"
		# nppnRaw=$((nppn))
		
	fi
	

	# PBSnodes="-l select=$nnodes:node_type=rome:mpiprocs=$nppn"
	PBSnodes="-l select=$nnodes:ncpus=$nppn:mpiprocs=$nppn"


	if [ -z "$queue" ]; then
		qsubqueue=""
	else
		qsubqueue="-queue $queue"
	fi			
	
	if [ $exclusive == true ]; then
		echo "WARNING: Exclusive not yet supported on this machine."
	fi	


	PBSMail=""
	if [ $mail == true ]; then 
		if [ -z "$UGSUBMIT_EMAIL" ]; then
			echo "please set UGSUBMIT_EMAIL or specify email with -email".
			exit
		fi 
		PBSMail="-M $UGSUBMIT_EMAIL -m $PBSMailtype"
	fi

	# interactive: qsub -IX -l walltime=00:50:00 -l mppwidth=4 -l mppnppn=4
			
	qsubargs="$qsubqueue -v UG4_ROOT -N $jobname -o job.output -e job.error -j oe -l walltime=$walltime $PBSnodes $PBSMail"

	echo "echo \"cd $outdir; $qsubrun\" | qsub $qsubargs" >> info.txt

	if [ $test == true ]; then
		echo "ONLY testing - NOT executing."
		return
	fi





	if [ $interactive == true ]; then
		echo "Interactive mode currently not supported on hermit. Aborting."
	else
		jobid=`echo "cd $outdir; $qsubrun" | qsub $qsubargs`
		return=$?	
		if [ ! $return == 0 ]; then
			echo "ERROR: qsub returned $return. Job has NOT been scheduled."
			exit
		fi
		jobid=`echo $jobid | sed 's/\([0-9]*\).*/\1/'`
	fi	
}



function UJS_GetOptions
{
	echo "WARNING: Unknown platform. guessed default values for MaxNumProcsPerNode (nppnmax) and MaxProcs (pemax) may be wrong."
	nppnmax=48
	pemax=1344
	nppnmaxSingleThread=48
}

function UJS_Info
{
	echo "Using $UGSUBMIT_TYPE"
	echo "qstat -u $USER"

	(
		echo "JOBID USERNAME QUEUE JOBNAME SessID ELAP-NDS TSK MEMORY TIME STATE TIME"
		# grep: get only jobs scheduled with ugsubmit
		qstat -u $USER | grep -E "job\.[0-9]+\.[0-9]+ *" | awk '
		BEGIN { OFS = "\t" }
		{
			# remove everything after first dot in jobname
			sub(/\..*$/, "", $1)

			status = $10
			if (status == "R" || status == "E") {
			status = "RUNNING"
			} else if (status == "Q" || status == "H" || status == "W" || status == "T" || status == "S") {
			status = "PENDING"
			} else {
			status = "PENDING"
			}
			print $1, $2, $3, $4, $5, $6, $7, $8, $9, status, $11
		}'  # change every status to RUNNING or PENDING, fallback to PENDING
	) | column -t  # format output as table
}


function UJS_Cancel
{
	echo "Using $UGSUBMIT_TYPE"
	echo "qdel $1"
	qdel $1
}
