systemctl enable mysql
systemctl start mysql


systemctl enable slurmctld
systemctl enable slurmdbd (enable the database)
systemctl enable slurmd (compute node)


sacctmgr add cluster compute-cluster
sacctmgr add account compute-account description="Compute accounts" Organization=OurOrg
sacctmgr create user myuser account=compute-account adminlevel=None


systemctl enable munge
systemctl restart munge


AccountingStorageTRES=gres/gpu
GresTypes=gpu
NodeName=linux1 Gres=gpu:8 ...


NodeName=slurm-node-0[0-1] Gres=gpu:2 CPUs=10 Sockets=1 CoresPerSocket=10 ThreadsPerCore=1 RealMemory=30000 State=UNKNOWN
PartitionName=compute Nodes=ALL Default=YES MaxTime=48:00:00 DefaultTime=04:00:00 MaxNodes=2 State=UP DefMemPerCPU=3000


CgroupAutomount=yes
CgroupReleaseAgentDir="/etc/slurm/cgroup"
ConstrainCores=yes
ConstrainDevices=yes
ConstrainRAMSpace=yes
#TaskAffinity=yes


Name=gpu File=/dev/nvidia0 CPUs=0-4
Name=gpu File=/dev/nvidia1 CPUs=5-9


$ srun --gres=gpu:1


$ sbatch runscript.sh


$ srun --pty -p test -t 10 --mem 1000 /bin/bash [script or app]


$ srun --pty -p test -t 10 --mem 1000 /bin/bash


$ scancel 999999


$ squeue -u akitzmiller


$ sacct -j 999999