#!/bin/bash
#
# LSF job submission script generated by MyCluster
#
# Job name
#BSUB -J {{my_name}}
# The batch system should use the current directory as working directory.
##BSUB -cwd
# Send status information to this email address.
#BSUB -u {{user_email}}
# Send me an e-mail when the job starts.
#BSUB -B
# Send me an e-mail when the job has finished.
#BSUB -N
# Redirect output stream to this file.
#BSUB -oo ./{{my_output}}.%J
# Which project should be charged
#BSUB -P {{project_name}}
# Queue name
#BSUB -q {{queue_name}}
# Number of tasks
#BSUB -n {{num_tasks}}
# Number of tasks per node
#BSUB -R "span[ptile={{tpn}}]"
{% if exclusive %}
# Exclusive node use
#BSUB -x
{% endif %}
# How much wallclock time will be required?
#BSUB -W {{wall_clock}}

export MYCLUSTER_QUEUE={{queue_name}}
export MYCLUSTER_JOB_NAME={{my_name}}
export NUM_TASKS={{num_tasks}}
export TASKS_PER_NODE={{tpn}}
export THREADS_PER_TASK={{num_threads_per_task}}
export NUM_NODES={{num_nodes}}

# OpenMP configuration
export OMP_NUM_THREADS=$THREADS_PER_TASK
export OMP_PROC_BIND=true
export OMP_PLACES=sockets

# OpenMPI
export OMPI_CMD="mpiexec -n $NUM_TASKS -npernode $TASKS_PER_NODE  {{openmpi_args}}"

# MVAPICH2
export MV2_CPU_BINDING_LEVEL=SOCKET
export MV2_CPU_BINDING_POLICY=scatter
export MVAPICH_CMD="mpiexec -n $NUM_TASKS -ppn $TASKS_PER_NODE -bind-to-socket"

# Intel MPI
# The following variables define a sensible pinning strategy for Intel MPI tasks -
# this should be suitable for both pure MPI and hybrid MPI/OpenMP jobs:
export I_MPI_PIN_DOMAIN=omp:compact # Domains are $$OMP_NUM_THREADS cores in size
export I_MPI_PIN_ORDER=scatter # Adjacent domains have minimal sharing of caches/sockets
#export I_MPI_FABRICS=shm:ofa
export IMPI_CMD="mpiexec -n $NUM_TASKS -ppn $TASKS_PER_NODE"
export I_MPI_HYDRA_BOOTSTRAP=lsf
export I_MPI_LSF_USE_COLLECTIVE_LAUNCH=1

# Summarise environment
echo -e "JobID: $LSB_JOBID\n======"
echo "Time: `date`"
echo "Running on master node: `hostname`"
echo "Current directory: `pwd`"

if [ "$LSB_DJOB_HOSTFILE" ]; then
        #! Create a machine file:
        cat $LSB_DJOB_HOSTFILE | uniq > machine.file.$LSB_JOBID
        echo -e "\nNodes allocated:\n================"
        echo `cat machine.file.$LSB_JOBID | sed -e 's/\..*$//g'`
fi

echo -e "\nnumtasks={{num_tasks}}, numnodes={{num_nodes}}, tasks_per_node={{tpn}} (OMP_NUM_THREADS=$OMP_NUM_THREADS)"

echo -e "\nExecuting command:\n==================\n{{my_script}}\n"

# Run user script
. {{my_script}}
exitcode=$?

# Report on completion
echo -e "\nJob Complete:\n==================\n"

echo -e "Complete with exit code $exitcode========\n"
exit $exitcode