Diagnose SLURM jobs not coming off queue

# List what you're running
# jobid, name, state, time, time limit, cpus, min memo, nodelist(reason)
squeue -u $USER -o "%.12i %.24j %.2t %.10M %.10l %.4C %.8m %R"

# Counts by state
squeue -u $USER -h -o "%T %j" | awk '{print $1}' | sort | uniq -c

# How long recent jobs took and their peak memory
sacct -u $USER -S now-2hours -P -n -o JobID,State,Elapsed,MaxRSS | \
    awk -F'|' '$2 == "COMPLETED" && $1 ~ /\./' | \
    sort -t'|' -k3 -r | head

# The full request for one job (time limit, CPUs, memory, whether it can be requeued, log paths):
scontrol show job <jobid>

# Scheduler weights for queueing the next job
# default_queue_depth sets how many jobs are queued by priority, after
# that, SLURM backfills in jobs
# bf_interval is how often the backfill check runs in seconds
# bf_max_job_user is how many jobs per user is checked during the backfill check
scontrol show config | grep -E "SchedulerParameters|PriorityWeight"

# Check priority for a job
sprio -l -j <jobid>

# Check your fair-share standing
# 1.0 is maximum
sshare -l -U -u $USER

# See what jobs are ahead of me
squeue -p defq -t PD --sort=-p -o "%.10i %.8u %.10Q %.5C %.8m %.10l %R" | \
    grep -E "\((Priority|Resources)\)" | head -15

# See what room is left
# %C: CPUs allocated, idle, other, total
# %N: node names
# %e: total free memory in MB
sinfo -p defq -o "%C"
sinfo -p defq -N -o "%N %C %e"