#!/bin/bash -l

#.Submit a job in MIT's Engaging cluster.

#SBATCH -J gkyl-bot               #.Name of the job (recommended 16 characters or less).

#SBATCH -p sched_mit_psfc         #.Request the PSFC queue.
#SBATCH --qos psfc_24h            #.Use this for >12h runs (PSFC queue). Remove for <12h runs.
#SBATCH -N 2                      #.Number of nodes to request.
#SBATCH -n 64                     #.Total number of cores (32 per node).
#SBATCH -t 06:00:00               #.Request wall time.

#SBATCH -o slurm-%j.out           #.Specify name format of output file.

#.Load modules (may not be needed).
module use /home/software/psfc/modulefiles
module add psfc/config
module load intel/2017-01
module load impi/2017-01
module load psfc/adios/1.13.1

export gComDir="$HOME/gkylsoft/gkyl/bin"      #.Full path to the gkyl executable.
export mpiComDir="$I_MPI_ROOT/intel64/bin"    #.Specify location of mpirun.

#.Cori printed workings about DVS strip width being 32.
#.Set it to 28 for now but some testing is needed (this may not be needed).
export MPICH_MPIIO_DVS_MAXNODES=28

#.Launch the MPI job (and print it to screen too).
echo $mpiComDir'/mpirun -n 64 '$gComDir'/gkyl input_file.lua'
$mpiComDir/mpirun -n 64 $gComDir/gkyl input_file.lua

exit 0