#!/bin/bash

#SBATCH -c 24
#SBATCH -N 1
#SBATCH -t 0-60:00
#SBATCH -p huce_intel
#SBATCH --constraint=intel
#SBATCH --mem=80000
#SBATCH --mail-type=END
#SBATCH -o stdout.%j

#------------------------------------------------------------------------------
#                  GEOS-Chem Global Chemical Transport Model                  !
#------------------------------------------------------------------------------
#BOP
#
# !MODULE: run_geos_chem
# 
# !DESCRIPTION: Runs a GEOS-Chem "classic" simulation on Odyssey.
#\\
#\\
# !REMARKS:
#  This script submits a typical 4x5 GEOS-Chem "classic" simulation 
#  (i.e.using OpenMP serial parallelization) on Odyssey.  A 4x5 fullchem 
#  simulation will take about 8 GB of RAM in total, across all CPUs.  (Your
#  simulation may use more memory, especially if you are saving out a lot
#  of diagnostic output.)  
#
#  The #SBATCH tags at the top of this script direct SLURM to reserve system
#  resources for your job.  Here is a list of commonly-used #SBATCH options:
#
#  -c 8                : Reserve 8 CPUs for the job.
#  -N 1                : Put all 8 CPUs on the same node.
#  -t 0-12:00          : Ask for 12 hours of CPU time (format = DD-HH:MM)
#  -p jacob            : Run job in the "jacob" queue.
#  --constraint=intel  : Only run on nodes with Intel CPUs
#  --mem=8000          : Ask for 8 GB of total memory for the job.
#  --mail-type=ALL     : Get notifications when the job starts, ends, or fails.
#  --mail-user=___     : Email address where notifications will get sent
#  -o stdout.%j        : Send stdout to this log file. 
#  -e stdout.%j        : Send stderr to this log file.
#
#  Alternative options for SBATCH tags:
#  (1) Instead of -c 8       you can also use --cpus-per-task=8
#  (2) Instead of -N 1       you can also use --nodes=1
#  (3) Instead of -t 0-12:00 you can also use -t 720 or -t 12:00
#
#  The available partitions (i.e. run queues) on Odyssey are:
#  (1) jacob ==> RESTRICTED TO JACOB-GROUP MEMBERS
#  (2) general 
#  (3) unrestricted 
#  (4) interactive 
#  (5) serial_requeue 
#  (6) bigmem
# 
#  Notes about stdout and stderr logs:
#  (1) If you omit -o, SLURM will send stdout to slurm-JOBID.out
#  (2) If you omit -e, SLURM will send stderr to slurm-JOBID.out
#  (3) Recommended: Use %j to have SLURM append the JOBID to the
#      stdout or stderr log file!  This is useful for debugging.
#  (4) If you omit --mail-user, SLURM will send email to the submitting user.
#
#  The available options for email notification (--mail-user) are:
#  (1) BEGIN : When the job starts
#  (2) END   : When the job finishes
#  (3) FAIL  : If the job dies with an error
#  (4) ALL   : (1), (2), and (3) combined
#
# !REVISION HISTORY: 
#  Use the "gitk" browser to view the Git version history!
#EOP
#------------------------------------------------------------------------------
#BOC

# Set the proper # of threads for OpenMP
# SLURM_CPUS_PER_TASK ensures this matches the number you set with -n above
export OMP_NUM_THREADS=$SLURM_CPUS_PER_TASK

# Run GEOS_Chem.  The "time" command will return CPU and wall times.
# Stdout and stderr will be directed to the log files specified above.
# NOTE: The "srun -c $OMP_NUM_THREADS" will tell SLURM that only
# the line where we run the GEOS_Chem executable should get multiple threads.
srun -c $OMP_NUM_THREADS time ./geos.mp


# Exit normally
exit 0
#EOC