Gaussian

#!/bin/bash
#==============================================================
# Gaussian16 Job Script — duhpc Cluster
# Version: 3.1  |  July 2026
# Department of Chemistry, University of Delhi
# Contact: Imran Ghani — ighani@ducc.du.ac.in
#
# FEATURES:
#   ✓ Auto-restart from checkpoint (uses Geom=AllCheck Guess=Read)
#   ✓ Walltime warning — saves checkpoint 5 min before kill
#   ✓ Auto-injects %Chk if user forgets
#   ✓ %NProcShared AND %Mem auto-matched to SLURM allocation
#   ✓ Memory/CPU coupling guard (MaxMemPerCPU on compute partition)
#   ✓ Pre-flight g16 environment check (clear error, not exit 127)
#   ✓ Smart failure diagnosis with recovery hints
#   ✓ Scratch preserved on failure for debugging
#
# USAGE:
#   1. Copy this file to your job directory
#   2. Edit JobFile= line with your input filename
#   3. Set MyEmail= to your institutional address (or leave blank)
#   4. Adjust --cpus-per-task and --mem to match your .com file
#   5. sbatch gaussian_job.sh
#
# PARTITION GUIDE:
#   compute → Gaussian, single-node, up to 5 days ← USE THIS
#   mpi     → GROMACS, NAMD, Amber MPI — multi-node jobs ONLY
#   short   → Test jobs, max 4 hours, max 8 CPUs
#   gpu     → Amber CUDA, GROMACS GPU — GPU jobs only
#
# ⚠ MEMORY RULE (IMPORTANT — reads "resources unavailable" otherwise):
#   The compute partition uses CR_CORE_MEMORY with MaxMemPerCPU=4200 MB.
#   Your job's real per-node core count = max(--cpus-per-task,
#   ceil(--mem_MB / 4200)). Requesting lots of RAM with few cores silently
#   forces MORE cores and can leave your job PENDING (Resources) even when the
#   status page shows free cores. Keep:  --mem(GB) ≤ cpus-per-task × 4
#     4 cores → ≤16G   8 cores → ≤32G   12 cores → ≤49G   16 cores → ≤65G
#==============================================================

#SBATCH --job-name=Gaussian
#SBATCH --partition=compute         # Always use compute for Gaussian
#SBATCH --nodes=1
#SBATCH --ntasks=1
#SBATCH --cpus-per-task=8           # Recommended: 4-16 (see guide below)
#SBATCH --mem=32G                   # Keep ≤ cpus-per-task × 4G (see MEMORY RULE)
#SBATCH --time=1-00:00:00           # Set a REALISTIC limit (max 5 days).
                                    # Shorter honest walltime = faster backfill.
#SBATCH --output=%x_%j.out          # Stdout (%x=jobname, %j=jobid)
#SBATCH --error=%x_%j.err           # Stderr
#SBATCH --signal=B:USR1@300         # Save checkpoint 5 min before walltime

# ══════════════════════════════════════════════════════════
# USER SETTINGS — Edit only this section
# ══════════════════════════════════════════════════════════
JobFile=molecule           # Input filename WITHOUT .com extension

MyEmail=""                 # Your institutional email for notifications, e.g.
                           #   MyEmail="yourusername@ducc.du.ac.in"
                           # LEAVE BLANK to disable mail. Do NOT use a public
                           # domain (gmail/yahoo/...) — cluster policy blocks
                           # them and they generate bounce-backs to the admin.

FreshStart=0               # Set to 1 to ignore any existing checkpoint and
                           # start a brand-new calculation from the .com file.
# ══════════════════════════════════════════════════════════

# CPU SELECTION GUIDE (pair with the MEMORY RULE above):
#   Geometry optimisation:  4-8  CPUs  --mem=16-32G
#   Single point (DFT):     8-12 CPUs  --mem=32-48G
#   Frequency calculation:  8-12 CPUs  --mem=32-48G
#   TD-DFT / NMR:           4-8  CPUs  --mem=16-32G
#   MP2 / CCSD(T):          8-16 CPUs  --mem=32-64G
#   Large molecule >100 at: 12-16 CPUs --mem=48-64G
#
# The script auto-matches %NProcShared and %Mem in your .com to this
# allocation, so you only need to get the #SBATCH lines right.

#==============================================================
# SETUP — do not edit below unless you know what you are doing
#==============================================================

set -o pipefail

SUBMIT_DIR=$SLURM_SUBMIT_DIR
CHK_PATH="$SUBMIT_DIR/${JobFile}.chk"
LOG_PATH="$SUBMIT_DIR/${JobFile}.log"
export GAUSS_SCRDIR=/scratch/gaussian/$USER/$SLURM_JOBID

# MaxMemPerCPU on the compute partition (MB). Keep in sync with slurm.conf.
MAX_MEM_PER_CPU=4200

#──────────────────────────────────────────────────────────
# EMAIL NOTIFICATION SETUP (opt-in, validated)
#──────────────────────────────────────────────────────────
# Note: --mail-type/--mail-user can only be set via sbatch, not mid-script.
# We validate here and advise; to actually enable mail, submit with:
#     sbatch --mail-type=BEGIN,END,FAIL --mail-user="$MyEmail" gaussian_job.sh
# or add matching #SBATCH lines AND set MyEmail above.
if [ -n "$MyEmail" ]; then
    case "$MyEmail" in
        *@gmail.com|*@yahoo.com|*@hotmail.com|*@outlook.com|*@ymail.com|your@email.com)
            echo "âš   MyEmail='$MyEmail' looks like a public/placeholder address."
            echo "   Cluster policy requires an institutional address"
            echo "   (yourusername@ducc.du.ac.in). Mail may be blocked/bounced." ;;
    esac
fi

#──────────────────────────────────────────────────────────
# WALLTIME TRAP — fires 5 minutes before SLURM kills job
#──────────────────────────────────────────────────────────
walltime_handler() {
    echo ""
    echo "╔══════════════════════════════════════════════════════╗"
    echo "║  ⚠  WALLTIME LIMIT APPROACHING — 5 minutes left!     ║"
    echo "╚══════════════════════════════════════════════════════╝"
    SAVED=0
    for chk in "$GAUSS_SCRDIR"/*.chk; do
        [ -f "$chk" ] && cp "$chk" "$SUBMIT_DIR/" && \
            echo "✓ Checkpoint saved: $SUBMIT_DIR/$(basename "$chk")" && SAVED=1
    done
    [ -f "$CHK_PATH" ] && echo "✓ Checkpoint in submit dir: $CHK_PATH" && SAVED=1
    [ $SAVED -eq 0 ] && \
        echo "✗ No checkpoint found — add %Chk=${JobFile}.chk to .com file"
    echo ""
    echo "TO RESTART: raise --time, then 'sbatch gaussian_job.sh'"
    echo "  (script auto-detects the checkpoint and continues)"
    echo "OR contact ighani@ducc.du.ac.in with Job ID $SLURM_JOB_ID"
}
trap 'walltime_handler' USR1

#──────────────────────────────────────────────────────────
# PRINT JOB HEADER
#──────────────────────────────────────────────────────────
echo "╔══════════════════════════════════════════════╗"
echo "║       Gaussian16 Job — duhpc Cluster         ║"
echo "╚══════════════════════════════════════════════╝"
echo "Job ID     : $SLURM_JOB_ID"
echo "Job Name   : $SLURM_JOB_NAME"
echo "User       : $USER"
echo "Node       : $SLURMD_NODENAME"
echo "CPUs       : $SLURM_CPUS_PER_TASK"
echo "Memory     : ${SLURM_MEM_PER_NODE:-unset} MB"
echo "Submit Dir : $SUBMIT_DIR"
echo "Input file : ${JobFile}.com"
echo "Started    : $(date)"
echo "══════════════════════════════════════════════"

#──────────────────────────────────────────────────────────
# MEMORY / CPU COUPLING GUARD
#──────────────────────────────────────────────────────────
MEM_MB=${SLURM_MEM_PER_NODE:-0}
if [ "$MEM_MB" -gt 0 ] && [ -n "$SLURM_CPUS_PER_TASK" ]; then
    CORES_FOR_MEM=$(( (MEM_MB + MAX_MEM_PER_CPU - 1) / MAX_MEM_PER_CPU ))
    if [ "$CORES_FOR_MEM" -gt "$SLURM_CPUS_PER_TASK" ]; then
        echo ""
        echo "âš   MEMORY/CPU NOTE:"
        echo "   --mem=${MEM_MB}MB needs ${CORES_FOR_MEM} cores at ${MAX_MEM_PER_CPU}MB/core,"
        echo "   but --cpus-per-task=${SLURM_CPUS_PER_TASK}. SLURM allocated ${CORES_FOR_MEM} cores"
        echo "   to satisfy the memory request — this can delay scheduling."
        echo "   Tip: keep --mem(GB) ≤ cpus-per-task × 4 to avoid this."
    fi
fi

#──────────────────────────────────────────────────────────
# LOAD GAUSSIAN  (set only g16root; let g16.profile do the rest)
#──────────────────────────────────────────────────────────
export g16root=/scratch/apps/gaussian
if [ ! -r "$g16root/g16/bsd/g16.profile" ]; then
    echo "ERROR: cannot read $g16root/g16/bsd/g16.profile on $SLURMD_NODENAME."
    echo "       Likely a group-membership / mount issue on this node."
    echo "       Contact ighani@ducc.du.ac.in (Job $SLURM_JOB_ID, node $SLURMD_NODENAME)."
    exit 1
fi
source "$g16root/g16/bsd/g16.profile"
ulimit -s unlimited 2>/dev/null || true

# Pre-flight: confirm g16 is actually runnable (turns exit-127 into a clear msg)
if ! command -v g16 >/dev/null 2>&1; then
    echo "ERROR: 'g16' not found on PATH after loading g16.profile."
    echo "       Node        : $SLURMD_NODENAME"
    echo "       g16root     : $g16root"
    echo "       Groups      : $(id -Gn 2>/dev/null)"
    echo "       Common cause: this node cannot resolve your gaussian-group"
    echo "       membership (GID 1016), or slurmd is holding a stale group cache."
    echo "       Admin fix   : verify 'id $USER' on the node; restart slurmd there."
    echo "       Contact ighani@ducc.du.ac.in with the above details."
    exit 127
fi
echo "g16 binary : $(command -v g16)"

#──────────────────────────────────────────────────────────
# SETUP SCRATCH
#──────────────────────────────────────────────────────────
mkdir -p "$GAUSS_SCRDIR" || { echo "ERROR: cannot create scratch $GAUSS_SCRDIR"; exit 1; }

#──────────────────────────────────────────────────────────
# VALIDATE INPUT FILE
#──────────────────────────────────────────────────────────
cd "$SUBMIT_DIR" || { echo "ERROR: cannot cd to $SUBMIT_DIR"; exit 1; }

if [ ! -f "${JobFile}.com" ]; then
    echo "ERROR: Input file '${JobFile}.com' not found in $SUBMIT_DIR"
    echo "Available .com files:"
    ls ./*.com 2>/dev/null || echo "  (none found)"
    exit 1
fi

sed -i 's/\r//g' "${JobFile}.com"   # strip Windows line endings

echo ""
echo "--- Input file settings ---"
grep -iE "^%NProcShared|^%Mem|^%Chk|^%nproc|^%mem|^%chk" "${JobFile}.com" || true

#──────────────────────────────────────────────────────────
# AUTO-MATCH %NProcShared TO ALLOCATION
#──────────────────────────────────────────────────────────
NPROC=$(grep -iE "^%NProcShared|^%nproc" "${JobFile}.com" | grep -o '[0-9]*' | head -1)
if [ -n "$NPROC" ] && [ "$NPROC" != "$SLURM_CPUS_PER_TASK" ]; then
    echo ""
    echo "⚠  %NProcShared=$NPROC ≠ --cpus-per-task=$SLURM_CPUS_PER_TASK — fixing .com"
    sed -i "s/NProcShared=$NPROC/NProcShared=$SLURM_CPUS_PER_TASK/Ig" "${JobFile}.com"
    sed -i "s/%nproc=$NPROC/%nproc=$SLURM_CPUS_PER_TASK/Ig" "${JobFile}.com"
    echo "   ✓ %NProcShared → $SLURM_CPUS_PER_TASK"
fi

#──────────────────────────────────────────────────────────
# AUTO-MATCH %Mem TO ALLOCATION  (target = --mem − 4 GB, floor 1 GB)
#──────────────────────────────────────────────────────────
if [ "$MEM_MB" -gt 0 ]; then
    TARGET_MEM_GB=$(( MEM_MB / 1024 - 4 ))
    [ "$TARGET_MEM_GB" -lt 1 ] && TARGET_MEM_GB=1
    CUR_MEM=$(grep -iE "^%mem" "${JobFile}.com" | head -1)
    if [ -n "$CUR_MEM" ]; then
        # Replace whatever %mem is present with the safe target in GB
        sed -i "s/^%[Mm][Ee][Mm]=.*/%Mem=${TARGET_MEM_GB}GB/" "${JobFile}.com"
        echo "⚠  %Mem set to ${TARGET_MEM_GB}GB (= --mem − 4G) to stay within allocation"
    else
        # Inject %Mem after the first Link0 line if absent
        sed -i "0,/^%/s//%Mem=${TARGET_MEM_GB}GB\n&/" "${JobFile}.com" 2>/dev/null || \
            sed -i "1i %Mem=${TARGET_MEM_GB}GB" "${JobFile}.com"
        echo "⚠  %Mem was missing — added %Mem=${TARGET_MEM_GB}GB"
    fi
fi

#──────────────────────────────────────────────────────────
# AUTO-INJECT %Chk IF MISSING
#──────────────────────────────────────────────────────────
if ! grep -qiE "^%chk" "${JobFile}.com"; then
    echo ""
    echo "⚠  No %Chk found — adding %Chk=${JobFile}.chk"
    TMPFILE=$(mktemp)
    echo "%Chk=${JobFile}.chk" > "$TMPFILE"
    cat "${JobFile}.com" >> "$TMPFILE"
    mv "$TMPFILE" "${JobFile}.com"
fi

#──────────────────────────────────────────────────────────
# CHECKPOINT / RESTART DECISION
#   Uses Geom=AllCheck Guess=Read so geometry, charge, multiplicity, title and
#   wavefunction all come FROM the checkpoint — no fragile coord/charge parsing.
#──────────────────────────────────────────────────────────
RESTART_MODE=0
ACTUAL_INPUT="${JobFile}.com"

# Guard against silently "restarting" an already-finished job
STALE_CHK=0
if [ -f "$CHK_PATH" ] && [ "$FreshStart" -eq 0 ]; then
    if [ -f "$LOG_PATH" ] && grep -q "Stationary point found" "$LOG_PATH" 2>/dev/null; then
        STALE_CHK=1
    fi
    if [ "${JobFile}.com" -nt "$CHK_PATH" ]; then
        STALE_CHK=1   # .com edited after the chk was written
    fi
fi

if [ "$FreshStart" -eq 1 ]; then
    echo ""
    echo "--- FreshStart=1 requested — ignoring any checkpoint ---"
    [ -f "$CHK_PATH" ] && echo "    (existing $CHK_PATH left untouched on disk)"
elif [ -f "$CHK_PATH" ] && [ "$STALE_CHK" -eq 1 ]; then
    echo ""
    echo "âš   Checkpoint exists but looks STALE (prior run converged, or .com edited)."
    echo "   NOT auto-restarting to avoid a 0-second no-op."
    echo "   → For a fresh run: set FreshStart=1 (or delete ${JobFile}.chk)."
    echo "   → To force continue from it anyway: remove this guard / rename the .log."
    echo "   Proceeding as a FRESH calculation from ${JobFile}.com."
elif [ -f "$CHK_PATH" ]; then
    echo ""
    echo "╔══════════════════════════════════════════════╗"
    echo "║  ✓ RESTART MODE — continuing from checkpoint  ║"
    echo "╚══════════════════════════════════════════════╝"
    echo "Checkpoint : $CHK_PATH ($(ls -lh "$CHK_PATH" | awk '{print $5}'))"

    MEM_LINE=$(grep -iE "^%mem"   "${JobFile}.com" | head -1)
    NPROC_LINE=$(grep -iE "^%nproc|^%nprocshared" "${JobFile}.com" | head -1)
    CHK_LINE=$(grep -iE "^%chk"   "${JobFile}.com" | head -1)
    ROUTE=$(grep -iE "^#" "${JobFile}.com" | head -1)

    # Add Restart to the route (Opt/IRC), else append Restart
    if echo "$ROUTE" | grep -qiE "Opt=\("; then
        NEW_ROUTE=$(echo "$ROUTE" | sed 's/Opt=(\([^)]*\))/Opt=(Restart,\1)/I')
    elif echo "$ROUTE" | grep -qiE "\bOpt\b"; then
        NEW_ROUTE=$(echo "$ROUTE" | sed 's/\bOpt\b/Opt=Restart/I')
    elif echo "$ROUTE" | grep -qiE "\bIRC\b"; then
        NEW_ROUTE=$(echo "$ROUTE" | sed 's/\bIRC\b/IRC=Restart/I')
    else
        NEW_ROUTE="$ROUTE Restart"
    fi
    # Pull geometry + charge/mult + title from the checkpoint; read prior WFN
    NEW_ROUTE="$NEW_ROUTE Geom=AllCheck Guess=Read"

    RESTART_INPUT="${JobFile}_restart_${SLURM_JOB_ID}.com"
    {
        echo "$CHK_LINE"
        [ -n "$MEM_LINE"   ] && echo "$MEM_LINE"
        [ -n "$NPROC_LINE" ] && echo "$NPROC_LINE"
        echo "$NEW_ROUTE"
        echo ""            # Geom=AllCheck: no title/charge/geometry blocks needed
    } > "$RESTART_INPUT"

    echo "--- Restart deck ---"; cat "$RESTART_INPUT"; echo "--------------------"
    ACTUAL_INPUT="$RESTART_INPUT"
    RESTART_MODE=1
else
    echo ""
    echo "--- Fresh start — no checkpoint found ---"
fi

#──────────────────────────────────────────────────────────
# RUN GAUSSIAN  (filename argument form; more reliable than stdin)
#──────────────────────────────────────────────────────────
echo ""
echo "--- Starting Gaussian16 ($([ $RESTART_MODE -eq 1 ] && echo RESTART || echo FRESH)) ---"
START=$(date +%s)

g16 "$ACTUAL_INPUT" > "$LOG_PATH" 2>&1
EXIT=$?

END=$(date +%s); ELAPSED=$((END-START))
printf -v WALL '%dh %dm %ds' $((ELAPSED/3600)) $(((ELAPSED%3600)/60)) $((ELAPSED%60))

#──────────────────────────────────────────────────────────
# SAVE CHECKPOINT
#──────────────────────────────────────────────────────────
echo ""; echo "--- Saving checkpoint ---"
CHK_SAVED=0
for chk in "$GAUSS_SCRDIR"/*.chk; do
    [ -f "$chk" ] && cp "$chk" "$SUBMIT_DIR/" && \
        echo "✓ Saved: $SUBMIT_DIR/$(basename "$chk")" && CHK_SAVED=1
done
[ -f "$CHK_PATH" ] && \
    echo "✓ Checkpoint: $CHK_PATH ($(ls -lh "$CHK_PATH" | awk '{print $5}'))" && CHK_SAVED=1
[ $CHK_SAVED -eq 0 ] && echo "✗ No checkpoint produced"

#──────────────────────────────────────────────────────────
# RESULT SUMMARY / DIAGNOSIS
#──────────────────────────────────────────────────────────
echo ""
echo "╔══════════════════════════════════════════════╗"
echo "â•‘                 JOB SUMMARY                   â•‘"
echo "╚══════════════════════════════════════════════╝"
echo "Job ID    : $SLURM_JOB_ID"
echo "Node      : $SLURMD_NODENAME"
echo "Wall time : $WALL"
echo "Exit code : $EXIT"

if [ ! -s "$LOG_PATH" ]; then
    echo "Status    : ✗ FAILED — Gaussian produced no output"
    echo "  Check the .err file and $ACTUAL_INPUT; scratch kept: $GAUSS_SCRDIR"
    echo "Finished  : $(date)"; exit 1
fi

if [ $EXIT -eq 0 ] && grep -q "Normal termination" "$LOG_PATH" 2>/dev/null; then
    echo "Status    : ✓ COMPLETED SUCCESSFULLY"
    echo ""; echo "--- Key results ---"
    grep -E "SCF Done|CCSD\(T\)|E\(MP2\)|Zero-point|Thermal correction|Dipole moment|Frequencies|Stationary point|Optimized Parameters|Normal termination|Job cpu time" \
        "$LOG_PATH" 2>/dev/null | grep -v "^$" | tail -12
    rm -rf "$GAUSS_SCRDIR"; echo ""; echo "Scratch   : Cleaned"

elif grep -qE "Erroneous write|galloc|Out of memory|sbrk|not enough memory" "$LOG_PATH" 2>/dev/null; then
    echo "Status    : ✗ FAILED — Memory error"
    CUR_GB=$(( MEM_MB / 1024 ))
    echo "  This job had --mem=${CUR_GB}G. Options:"
    echo "   • Raise --mem AND cpus-per-task together (keep --mem ≤ cores×4G),"
    echo "     e.g. --cpus-per-task=$((SLURM_CPUS_PER_TASK+4)) --mem=$(( (SLURM_CPUS_PER_TASK+4)*4 ))G"
    echo "   • The script will auto-set %Mem to (--mem − 4G) next run."
    echo "  Scratch kept: $GAUSS_SCRDIR"

elif grep -qE "Convergence failure|SCF has not converged" "$LOG_PATH" 2>/dev/null; then
    echo "Status    : ✗ FAILED — SCF convergence"
    echo "  Add to route card:  SCF=(MaxCycles=512,XQC)"
    echo "  Scratch kept: $GAUSS_SCRDIR"

elif grep -qE "Optimization stopped|Berny optimization" "$LOG_PATH" 2>/dev/null && \
     ! grep -q "Stationary point found" "$LOG_PATH" 2>/dev/null; then
    echo "Status    : ✗ INCOMPLETE — geometry not converged"
    [ $CHK_SAVED -eq 1 ] && echo "  Checkpoint saved — resubmit to continue: sbatch gaussian_job.sh"
    echo "  Scratch kept: $GAUSS_SCRDIR"

else
    echo "Status    : ✗ FAILED"
    echo ""; echo "Last 15 lines of log:"; tail -15 "$LOG_PATH" 2>/dev/null
    [ $CHK_SAVED -eq 1 ] && echo "Checkpoint saved — resubmit: sbatch gaussian_job.sh"
    echo "Scratch kept: $GAUSS_SCRDIR"
fi

echo ""
echo "Finished  : $(date)"
echo "══════════════════════════════════════════════"
echo "Log: $LOG_PATH   |   Help: ighani@ducc.du.ac.in"
exit $EXIT