#!/bin/bash
#
# PBS job submission script generated by MyCluster
#
# Job name
#PBS -N {{my_name}}
# Join the error stream to the output stream.
#PBS -j oe
# Send status information to this email address.
#PBS -M {{user_email}}
# Send me an e-mail when the job has finished.
#PBS -m abe
# Queue name
#PBS -q {{queue_name}}
# Project name
#PBS -A {{project_name}}
# Maximum wall clock
#PBS -l walltime={{wall_clock}}
# Request number of nodes
#PBS -l select={{num_nodes}}

export MYCLUSTER_QUEUE={{queue_name}}
export MYCLUSTER_JOB_NAME={{my_name}}
export NUM_TASKS={{num_tasks}}
export TASKS_PER_NODE={{tpn}}
export THREADS_PER_TASK={{num_threads_per_task}}
export NUM_NODES={{num_nodes}}

# OpenMP configuration
export OMP_NUM_THREADS=$THREADS_PER_TASK
export OMP_PROC_BIND=true
export OMP_PLACES=sockets

# OpenMPI
export OMPI_CMD="mpiexec -n $NUM_TASKS -npernode $TASKS_PER_NODE {{openmpi_args}}"

# MVAPICH2
export MV2_CPU_BINDING_LEVEL=SOCKET
export MV2_CPU_BINDING_POLICY=scatter
export MVAPICH_CMD="mpiexec -n $NUM_TASKS -ppn $TASKS_PER_NODE -bind-to-socket"

# Intel MPI
# The following variables define a sensible pinning strategy for Intel MPI tasks -
# this should be suitable for both pure MPI and hybrid MPI/OpenMP jobs:
export I_MPI_PIN_DOMAIN=omp:compact # Domains are $$OMP_NUM_THREADS cores in size
export I_MPI_PIN_ORDER=scatter # Adjacent domains have minimal sharing of caches/sockets
#export I_MPI_FABRICS=shm:ofa
export IMPI_CMD="mpiexec -n $NUM_TASKS -ppn $TASKS_PER_NODE"

cd $PBS_O_WORKDIR

# Summarise environment
echo -e "JobID: $PBS_JOBID\n======"
echo "Time: `date`"
echo "Running on master node: `hostname`"
echo "Current directory: `pwd`"

if [ "$PBS_NODEFILE" ]; then
        #! Create a machine file:
        cat $PBS_NODEFILE | awk '{print $1, " slots=" $2}' > machine.file.$PBS_JOBID
        echo -e "\nNodes allocated:\n================"
        echo `cat machine.file.$PBS_JOBID | sed -e 's/\..*$//g'`
fi

echo -e "\nnumtasks={{num_tasks}}, numnodes={{num_nodes}}, tasks_per_node={{tpn}} (OMP_NUM_THREADS=$OMP_NUM_THREADS)"

echo -e "\nExecuting command:\n==================\n{{my_script}}\n"

# Run user script
. {{my_script}}
exitcode=$?

echo -e "Complete with exit code $exitcode========\n"
exit $exitcode
