#!/bin/bash
######################################################################
#                     qsub on Hermit / HLRS (aprun)
#        qsub on NecNehalem / NEC Nehalem Cluster / HLRS (mpirun)
#-------------------------------------------------------------------
######################################################################

function UJS_Submit
{
	UJS_GetOptions

	if [ $walltime == "unlimited" ]; then
		walltime=01:00:00
	fi
	
	if [ $((npe%nppn)) -ne 0 ]; then
		echo "NOTE: npe=$npe is not dividable by nppn=$nppn. rounding up"
	fi
	if [ $UGSUBMIT_TYPE == "NecNehalem" ]; then
		echo "Cluster: NecNehalem / NEC Nehalem Cluster / HLRS. Scheduler: qsub."
		PBSnodes="-l nodes=$nnodes:nehalem:ppn=$nppn"
		qsubrun="mpirun -np $npe $executable $args"
		
		if [ -z "$queue" ]; then
			qsubqueue="-q user"
		else
			qsubqueue="-q $queue"
		fi		
	else
		echo "Cluster: Hermit/Cray XE6. Scheduler: qsub"
		if [ $nppn -gt $nppnmaxSingleThread ]; then
			qsubrun="aprun -n $npe -N $nppn -j2 $executable $args"
			#qsub doesn't know about hyperthreading. Only specify used procs. Round up.
			nppnRaw=$(((nppn+1)/2))
		else
			qsubrun="aprun -n $npe -N $nppn -j1 $executable $args"
			nppnRaw=$((nppn))
		fi
		PBSnodes="-l nodes=$nnodes:ppn=$nppnRaw"

		if [ -z "$queue" ]; then
			qsubqueue=""
		else
			qsubqueue="-q $queue"
		fi			
	fi
	
	if [ $exclusive == true ]; then
		echo "WARNING: Exclusive not yet supported on this machine."
	fi	

	pbsMail=""
	if [ $mail == true ]; then 
		if [ -z "$UGSUBMIT_EMAIL" ]; then
			echo "please set UGSUBMIT_EMAIL or specify email with -email".
			exit
		fi 
		pbsMail="-M $UGSUBMIT_EMAIL -m $pbsMailtype"
	fi

	# interactive: qsub -IX -l walltime=00:50:00 -l mppwidth=4 -l mppnppn=4
			
	qsubargs="$qsubqueue -v UG4_ROOT -N $jobname -o job.output -e job.error -j oe -l walltime=$walltime $PBSnodes $pbsMail"

	echo "echo \"cd $outdir; $qsubrun\" | qsub $qsubargs" >> info.txt

	if [ $test == true ]; then
		echo "ONLY testing - NOT executing."
		return
	fi

	if [ $interactive == true ]; then
		echo "Interactive mode currently not supported on hermit. Aborting."
	else
		jobid=`echo "cd $outdir; $qsubrun" | qsub $qsubargs`
		return=$?	
		if [ ! $return == 0 ]; then
			echo "ERROR: qsub returned $return. Job has NOT been scheduled."
			exit
		fi
		jobid=`echo $jobid | sed 's/\([0-9]*\).*/\1/'`
	fi	
}



function UJS_GetOptions
{
	if [ $UGSUBMIT_TYPE == "NecNehalem" ]; then
		nppnmax=4
		pemax=1024
		nppnmaxSingleThread=4
	elif [ $UGSUBMIT_TYPE == "Hazelhen" ]; then
		nppnmax=48
		pemax=370176
		nppnmaxSingleThread=24
	elif [ $UGSUBMIT_TYPE == "Hornet" ]; then
		nppnmax=24
		pemax=94656
		nppnmaxSingleThread=24
	elif [ $UGSUBMIT_TYPE == "Hermit" ]; then
		nppnmax=32
		pemax=113664
		nppnmaxSingleThread=32
 	elif [ $UGSUBMIT_TYPE == "Hawk" ]; then
                nppnmax=64
                pemax=720896
                nppnmaxSingleThread=64
	else
		echo "WARNING: Unknown platform. guessed default values for MaxNumProcsPerNode (nppnmax) and MaxProcs (pemax) may be wrong."
		nppnmax=32
		pemax=113664
		nppnmaxSingleThread=32
	fi
}

function UJS_Info
{
	echo "Using $UGSUBMIT_TYPE"
	echo "qstat -u $USER -r"
	qstat -u $USER	
}


function UJS_Cancel
{
	echo "Using $UGSUBMIT_TYPE"
	echo "qdel $1"
	qdel $1
}
