#!/bin/bash
#
# SLURM job submission script generated by MyCluster
#
# Job name
#SBATCH -J {{my_name}}
# Send status information to this email address.
#SBATCH --mail-user={{user_email}}
# Send me an e-mail when the job has finished.
#SBATCH --mail-type=ALL
# Redirect output stream to this file.
#SBATCH --output {{my_output}}.%j
# Which project should be charged
#SBATCH -A {{project_name}}
# Partition name
#SBATCH -p {{queue_name}}
# Number of nodes
#SBATCH --nodes {{num_nodes}}
# Number of tasks
#SBATCH --ntasks {{num_tasks}}
{% if exclusive %}
# Exclusive node use
#SBATCH --exclusive
{% endif %}
# Do not requeue job on node failure
#SBATCH --no-requeue
# High performance cpu governor
#SBATCH --cpu-freq=Performance
# How much wallclock time will be required?
#SBATCH --time={{wall_clock}}
{% if qos %}
#SBATCH --qos {{qos}}
{% endif %}
{% if gres %}
#SBATCH --gres={{gres}}
{% endif %}


export MYCLUSTER_QUEUE={{queue_name}}
export MYCLUSTER_JOB_NAME={{my_name}}
export NUM_TASKS={{num_tasks}}
export TASKS_PER_NODE={{tpn}}
export THREADS_PER_TASK={{num_threads_per_task}}

{% if num_threads_per_task < 0 %}
CPUS_PER_NODE=`echo ${SLURM_JOB_CPUS_PER_NODE}  | cut -d \( -f 1`
export THREADS_PER_TASK=`expr ${CPUS_PER_NODE} / ${TASKS_PER_NODE}`
{% endif %}

export NUM_NODES={{num_nodes}}
export JOBID=$SLURM_JOB_ID

# OpenMP configuration
export OMP_NUM_THREADS=$THREADS_PER_TASK
export OMP_PROC_BIND=true
export OMP_PLACES=sockets

# OpenMPI
export OMPI_CMD="mpiexec -n $NUM_TASKS -npernode $TASKS_PER_NODE {{openmpi_args}}"
# OpenMPI 1.10
export OMPI_NEW_CMD="mpiexec -n $NUM_TASKS --map-by ppr:1:numa --bind-to numa"

# MVAPICH2
export MV2_CPU_BINDING_LEVEL=SOCKET
export MV2_CPU_BINDING_POLICY=scatter
export MVAPICH_CMD="mpiexec -n $NUM_TASKS -ppn $TASKS_PER_NODE -bind-to-socket"

# Intel MPI
# The following variables define a sensible pinning strategy for Intel MPI tasks -
# this should be suitable for both pure MPI and hybrid MPI/OpenMP jobs:
export I_MPI_PIN_DOMAIN=omp:compact # Domains are $OMP_NUM_THREADS cores in size
export I_MPI_PIN_ORDER=scatter # Adjacent domains have minimal sharing of caches/sockets
#export I_MPI_FABRICS=shm:ofa
#export I_MPI_FABRICS=shm:tmi
#export TMI_CONFIG=<path_to_impi>/intel64/etc/tmi.conf
export IMPI_CMD="mpiexec -n $NUM_TASKS -ppn $TASKS_PER_NODE"

# Summarise environment
echo -e "JobID: $JOBID\n======"
echo "Time: `date`"
echo "Running on master node: `hostname`"
echo "Current directory: `pwd`"

if [ "$SLURM_JOB_NODELIST" ]; then
        #! Create a machine file:
        echo $SLURM_JOB_NODELIST | uniq > machine.file.$JOBID
        echo -e "\nNodes allocated:\n================"
        echo `cat machine.file.$JOBID | sed -e 's/\..*$//g'`
fi

echo -e "\nnumtasks={{num_tasks}}, numnodes={{num_nodes}}, tasks_per_node={{tpn}} (OMP_NUM_THREADS=$OMP_NUM_THREADS)"

echo -e "\nExecuting command:\n==================\n{{my_script}}\n"

# Run user script
. {{my_script}}
exitcode=$?

# Report on completion
echo -e "\n Job Complete with exit code $exitcode========\n"
exit $exitcode
