slurm.conf(5) (NAME) slurm.conf - (DESCRIPTION) slurm.conf (ASCII) Slurm . . SLURM_CONF . Slurm <<-f>> (override) . (case insensitive). <<#>> () . Slurm SIGHUP <> . TCP . <> (whitespace) . ( INCLUDE MODIFIERS ). : slurm.conf Slurm Slurm . slurm.conf / <> . <> <> . Slurm FILE AND DIRECTORY PERMISSIONS . (PARAMETERS) : AccountingStorageBackupHost . accounting_storage/slurmdbd slurmdbd . SlurmDBD . AccountingStorageEnforce (association-based) . : all nojobs nosteps . associations (association) . limits . associations . nojobs Slurm (step) . nosteps . nosteps Slurm . qos qos . associations . safe qos TRES-minutes . TRES-minutes . (kill) . qos . limits associations . wckeys (workload characterization key) . associations TrackWCKey ( ) . AccountingStorageExternalHost slurmdbd ([:port][,...]) . AccountingStoragePort . slurmdbd --cluster/-M . slurmdbd . slurmdbd slurmctld slurmdbd . AccountingStorageHost . SlurmDBD . AccountingStorageParameters . max_step_records= -- batch extern interactive. - (key-value) : SSL_CERT . SSL_CA (CA). SSL_CAPATH SSL CA. SSL_KEY . SSL_CIPHER SSL. AccountingStoragePass . . SlurmDBD ( ) MUNGE MUNGE MUNGE . AccountingStoragePass MUNGE ( <>). NULL . AccountingStoragePort . . SLURMDBD_PORT . 6819 . DbdPort slurmdbd.conf . AccountingStorageTRES . sbatch/srun . GRES BB ( burst buffer) (license) CPU Memory Node Energy FS/[Disk|Lustre] IC/OFED Pages VMem . Billing CPU Energy Memory Node FS/Disk Pages VMem . TRES . AccountingStorageTRES=gres/craynetwork,license/iop1 billing cpu energy memory nodes fs/disk pages vmem gres craynetwork iop1 . . TRES slurmctld . GRES ( GPU ) . "AccountingStorageTRES=gres/gpu,gres/gpu:tesla,gres/gpu:volta" <> <> GPU <> (GPU) (<> <> GPU ) . GRES ( "gres/gpu:tesla,gres/gpu:volta") ( "gres/gpu") . . GRES . job_submit GRES . NOTE: gres/gpu gres/gpumem gres/gpuutil . gres/gpu gres/gpumem gres/gpuutil . AccountingStorageType . . : accounting_storage/slurmdbd SlurmDBD MySQL . <> . AccountingStoreFlags slurmctld . : job_comment comment Accounting Storage. AdminComment SystemComment . job_env (batch job) Accounting Storage. job_extra extra Accounting Storage. job_script (batch script) Accounting Storage. no_stdio stdio . StdOut StdErr StdIn . AcctGatherNodeFreq AcctGather . none AcctGather . . acct_gather_energy/rapl 300 (overflow). . . : JobAcctGatherFrequency . AcctGatherEnergyType . jobacct_gather slurmd . (exclusive job allocation) . ( sstat sacct) . . : acct_gather_energy/gpu GPU ( rsmi) . rsmi . : slurmd gpu . acct_gather_energy/ipmi (BMC) (IPMI) . acct_gather_energy/pm_counters (BMC) HPE Cray . acct_gather_energy/rapl (RAPL) . RAPL <> . acct_gather_energy/xcc Lenovo SD650 XClarity (XCC) IPMI OEM . AcctGatherInterconnectType (interconnect) . jobacct_gather slurmd . . ( sstat sacct) . : acct_gather_interconnect/ofed Infiniband Infiniband OFED . TRES "ic/ofed" AccountingStorageTRES . acct_gather_interconnect/sysfs sysfs acct_gather.conf(5) . TRES "ic/sysfs" AccountingStorageTRES . AcctGatherFilesystemType . jobacct_gather slurmd . . ( sstat sacct) . : acct_gather_filesystem/lustre Lustre /proc/fs/lustre/ . lustre TRES "fs/lustre" AccountingStorageTRES . AcctGatherProfileType (detailed job profiling) . jobacct_gather slurmd I/O . . . : acct_gather_profile/hdf5 HDF5 . acct_gather.conf . acct_gather_profile/influxdb influxdb . influxdb (retention policy) acct_gather.conf . AllowSpecResourcesUsage <> Slurm CoreSpecCount (override) . --core-spec . Cray <> <> . AuthAltTypes slurmctld . auth/jwt . NOTE: AuthAltParameters jwt_hs256.key slurmctld StateSaveLocation . jwt_hs256.key SlurmUser root . jwt_hs256.key slurmctld slurmdbd . auth/jwt SLURM_JWT . AuthType . AuthAltParameters . . disable_token_creation <> SlurmUser. max_token_lifespan= ( ) . SlurmUser . scontrol token JWT / . jwks= JWKS. SlurmUser root SlurmUser 0400 . 'other' . RS256 . HS256 ( ) jwt_key HS256 ( ) . jwt_key= JWT. HS256 . SlurmUser root SlurmUser 0400 . 'other' . jwt_hs256.key StateSaveLocation . use_jwt_client_ids JWT . JWT uid gid . (claim) (id) JSON gecos dir shell gids () groups () . JWT slurmctld . ( auth/jwt.) use_jwt_client_ids_only JWT auth/jwt . use_jwt_client_ids . ( auth/jwt.) userclaimfield= UserName Slurm (sun). Slurm . ( grant .) : () AuthInfo Slurm ( slurmctld slurmd) Slurm. AuthType . . . cred_expire (credential) ( "cred_expire=1200"). prolog (page out) slurmd . (requeued) . 120 . socket MUNGE ( "socket=/var/run/munge/munge.socket.2"). "/var/run/munge/munge.socket.2" . auth/munge cred/munge . ttl ( "ttl=300"). AuthType . auth/munge MUNGE 300 . auth/slurm 60 . auth/jwt 1800 . use_client_ids auth/slurm LDAP . nss_slurm slurmstepd . (login nodes) LDAP/OS . . NOTE: Slurm use_client_ids : o AllowGroups o Users o AllowUserBoot helpers.conf(5) AuthType Slurm. Slurm AuthType . . . : auth/munge MUNGE (). ( "https://dun.github.io/munge/" ). auth/slurm Slurm. BatchStartTimeout ( ) (batch job) . 10 () . Prolog slurmd (paged) . NOTE: Slurm slurmctld (head node) . BatchStartTimeout . srun Prolog . BcastExclude (broadcast) (shared object) sbcast srun --bcast . <> . <> . sbcast --exclude srun --bcast-exclude . BcastParameters sbcast srun --bcast . . : DestDir= (broadcast) . --chdir srun. Compression= . "lz4" "none" . sbcast --compress "lz4" "none" . . send_libs . . srun LD_LIBRARY_PATH (cache) . --send-libs sbcast srun . . BurstBufferType (burst buffers). . : burst_buffer/datawarp Cray DataWarp API . burst_buffer/lua (hooks) API Lua . ( file staging ) . CertgenParameters certgen . : certgen_script= TLS. keygen_script stdin PEM stdout . 0 . keygen_script= . PEM stdout stdin certgen_script . 0 . CertgenType certgen . : certgen/script / . CertmgrParameters certmgr . certificate_renewal_period= slurmd/sackd ( ) slurmctld . 1440 ( ). generate_csr_script= (CSR). . PEM stdout 0 . . certmgr/script . . get_node_cert_key_script= CSR generate_csr_script . . ( PEM) stdout . 0 . certmgr/script . . get_node_token_script= slurmctld validate_node_script= . . stdout 0 . . certmgr/script . . sign_csr_script= . validate_node_script= . ( generate_csr_script= ) . stdout 0 . . certmgr/script . slurmctld . single_use_tokens ( scontrol) ( ). validate_node_script= . . 0 . . certmgr/script . slurmctld . CertmgrType TLS slurmd/sackd. certmgr/script (script hooks) . CertmgrParameters . CliFilterParameters cli_filter. . : cli_filter_lua_path= cli_filter.lua cli_filter/lua. ( slurm.conf). NOTE: cli_filter.lua 755 644 SlurmdUser . CliFilterPlugins / . . cli_filter . : cli_filter/lua cli_filter lua . cli_filter/syslog . salloc/sbatch/srun JSON syslog . slurmctld . cli_filter/user_defaults $HOME/.slurm/defaults key=value key salloc/sbatch/srun value . : time=1:30 mem=2048 "-t 1:30" "--mem=2048" . ClusterName Slurm . . . . MySQL/MariaDB . CommunicationParameters . block_null_hash Slurm (20.11.9 21.08.8) . Slurm 20.11.9/21.08.8 . disable_http slurmctld slurmd HTTP. host_unreach_retry_count= connect() connect() EHOSTUNREACH . Slurm . Slurm . ( EHOSTUNREACH Slurm ). DisableIPv4 IPv4 slurm ( slurmdbd). slurmdbd.conf . EnableIPv6 IPv6 slurm ( slurmdbd). IPv4 IPv6 /etc/gai.conf . slurmdbd.conf . getnameinfo_cache_timeout munge AuthType slurmctld getnameinfo IP munge . slurmctld IP () . 0 . 60 . keepaliveinterval= keepalive (idle) . srun slurmstepd slurmdbd . . keepaliveprobes= keepalive . srun slurmstepd slurmdbd . . keepalivetime= ( ) keepalive . srun slurmstepd slurmdbd . . keepalive . NoCtldInAddrAny (bind) slurmctld . NoInAddrAny (bind) . / slurmctld . CompleteWait COMPLETING . . (fragmentation). . COMPLETING Epilog . ( ). KillWait . KillWait . CompleteWait . 65533 . COMPLETING . Epilog COMPLETING CompleteWait . NOTE: reduce_completing_frag CompleteWait . CpuFreqDef CPU (job step) --cpu-freq . : Conservative CPU Conservative OnDemand CPU OnDemand Performance CPU Performance PowerSave CPU PowerSave : . --cpu-freq . CpuFreqGovernors CPU --cpu-freq salloc sbatch srun . : Conservative CPU Conservative OnDemand CPU OnDemand ( ) Performance CPU Performance ( ) PowerSave CPU PowerSave SchedUtil CPU SchedUtil UserSpace CPU UserSpace ( ) : OnDemand Performance UserSpace. CredType . : cred/munge Munge (). cred/slurm Slurm. DataParserParameters= data_parser. --json --yaml sacct(1) scontrol(1) sinfo(1) squeue(1) sacctmgr(1) sdiag(1) sshare(1) . : data_parser . DebugFlags . . DebugFlags SlurmctldDebug 'verbose' . . NOTE: SLURM_DEBUG_FLAGS ( ) . slurm.conf . : Accrue (Accrue counters) Agent RPC (RPC Slurm) AuditRPCs RPC slurmctld RPC . AuditTLS TLS Backfill (Backfill scheduler) BackfillMap . Backfill . BurstBuffer (Burst Buffer) Cgroup Cgroup ConMgr (Connection manager) CPU_Bind CPU CpuFrequency CPU --cpu-freq. Data . DBD_Agent RPC (RPC DBD) Dependency Elasticsearch Elasticsearch ( ). JobComp. Energy AcctGatherEnergy Federation Gres (Generic resource) Hetjob (Heterogeneous job) Gang (Gang scheduling) GLOB_SILENCE "*" conf . JobAccountGather ( ). JobComp (Job Completion) License (License) Namespace (Namespace) Metrics (Metrics) Network . Warning: . NetworkRaw . Warning: . NodeFeatures (Node Features) NO_CONF_HASH slurm.conf Slurm Power ( / suspend/resume) Priority Profile AcctGatherProfile Protocol Reservation (Advanced reservations) Route Script Slurm. slurmctld PrologSlurmctld EpilogSlurmctld . SelectType Steps slurmctld Switch thread () . TLS TLS TraceJobs slurmctld. . Triggers slurmctld DefCpuPerGPU CPU GPU . --cpus-per-task --cpus-per-gpu . DefMemPerCPU CPU . (over-subscribing) (paging) . (SelectType=select/cons_tres) DefMemPerCPU . 0 () . DefMemPerGPU DefMemPerNode MaxMemPerCPU . DefMemPerCPU DefMemPerGPU DefMemPerNode . NOTE: CPU . (thread) . --threads-per-core ( --hint=nomultithread --threads-per-core=1 ) CPU . CPU . . --threads-per-core . . cpu : $ salloc -n3 --mem-per-cpu=100 salloc: Granted job allocation 17199 $ sacct -j $SLURM_JOB_ID -X -o jobid%7,reqtres%35,alloctres%35 JobID ReqTRES AllocTRES ------- ----------------------------------- ----------------------------------- 17199 billing=3,cpu=3,mem=300M,node=1 billing=4,cpu=4,mem=400M,node=1 --threads-per-core=1 . CPU . cpu . $ salloc -n3 --mem-per-cpu=100 --threads-per-core=1 salloc: Granted job allocation 17200 $ sacct -j $SLURM_JOB_ID -X -o jobid%7,reqtres%35,alloctres%35 JobID ReqTRES AllocTRES ------- ----------------------------------- ----------------------------------- 17200 billing=3,cpu=3,mem=300M,node=1 billing=6,cpu=6,mem=300M,node=1 DefMemPerGPU GPU . 0 () . (best effort) GPU MaxMemPerNode . DefMemPerCPU DefMemPerNode . DefMemPerCPU DefMemPerGPU DefMemPerNode . DefMemPerNode . . (SelectType=select/linear) (OverSubscribe=yes OverSubscribe=force) DefMemPerNode . 0 () . DefMemPerCPU DefMemPerGPU MaxMemPerCPU . DefMemPerCPU DefMemPerGPU DefMemPerNode . DependencyParameters . disable_remote_singleton . . . kill_invalid_depend JOB_CANCELLED . DependencyNeverSatisfied . max_depend_depth= . . . DisableRootJobs "YES" root . "NO" root . DisableRootJobs . EioTimeout srun slurmstepd TCP/IP srun . . 65533 . EnforcePartLimits . MaxMemPerCPU MaxMemPerNode MinNodes MaxNodes MaxTime AllocNodes AllowAccounts AllowGroups AllowQOS QOS. . QOS . ALL / . . ANY . NO . . . . Epilog root ( "/usr/local/slurm/epilog"). ( ) slurm.conf . glob ( glob (7) ) epilog ( "/etc/slurm/epilog.d/*"). epilog (z-a -> Z-A -> 9-0) . () Epilog . epilog . Prolog and Epilog Scripts . Epilog COMPLETING . COMPLETING Epilog . NOTE: epilog . EpilogMsgTime slurmctld epilog slurmd . RPC . 2000 . 1000000 ( ) . . epilog . EpilogSlurmctld slurmctld ( "/usr/local/slurm/epilog_controller"). SlurmUser ( scontrol(1) ). . . Prolog and Epilog Scripts . NOTE: epilog . EpilogTimeout Slurm Epilog . PrologEpilogTimeout . Epilog slurmd EpilogSlurmctld slurmctld epilog SPANK slurm_spank_job_epilog . Epilog slurm_spank_job_epilog (time out) (drained) . (log) . FairShareDampeningFactor (fair share) . ( 1 10 100 5 ). 1 . FederationParameters . . fed_display ( squeue sinfo sprio ) . --federation . --local . slurmdbd slurmctld --cluster . FirstJobId (job id) Slurm . 1 . 0 . 1 . MaxJobId GresTypes (generic resources) ( GresTypes=gpu,mps). GRES . . . GroupUpdateForce /etc/group . /etc/group . 1 . GroupUpdateTime . GroupUpdateTime (cache) . 600 . . GroupUpdateForce . GpuFreqDef=[[,] GPU (job step) --gpu-freq . GPU . . --gpu-freq GPU . GPU . (power caps) . type "memory" . type GPU . value "low" "medium" "high" "highm1" (MHz) . . . "GpuFreqDef=medium,memory=high" "GpuFreqDef=450" . (value) : low . medium . high . highm1 (high ) . HashPlugin . : hash/k12 KangarooTwelve . . hash/sha3 SHA-3 . NOTE: HashPlugin slurm.conf slurmdbd.conf . HealthCheckInterval HealthCheckProgram. . HealthCheckNodeState HealthCheckProgram . *_ONLY. ANY . ALLOC ALLOC ( CPU ) . ANY . CYCLE HealthCheckInterval . . IDLE IDLE () . NONDRAINED_IDLE IDLE DRAINED ( ) . MIXED MIXED ( CPU ) . REBOOT_ONLY (reboot) . . START_ONLY slurmd . . HealthCheckProgram root NOT_RESPONDING . (DRAIN) . ( "scontrol update NodeName=foo State=drain Reason=tmp_file_system_full" ). HealthCheckInterval . HealthCheckProgram . 60 (kill) . slurmd slurmctld . HealthCheckNodeState START_ONLY REBOOT_ONLY slurmd slurmd . . HealthCheckTimeout HealthCheckProgram . 60 . HealthCheckTimeout HealthCheckInterval . 0 . 65535 . HttpParserType http_parser . http_parser/libhttp_parser http_parser/llhttp_parser . : http_parser/libhttp_parser libhttp_parser. http_parser/llhttp_parser llhttp. InactiveLimit (non-responsive) ( srun salloc) . . (batch jobs) . srun srun (stopped) . RootOnly ( root ). () 65533 . InteractiveStepOptions LaunchParameters=use_interactive_step salloc srun InteractiveStepOptions . "--interactive --preserve-env --pty $SHELL" . "--interactive" srun . InteractiveStepOptions << >> (interactive step) . JobAcctGatherType JobAcctGather CPU / (I/O) (interconnect) GPU (task) Slurm . . (unset) . : jobacct_gather/cgroup () CPU cgroup ( memory.stat cpu.stat) CgroupPlugin ( "man cgroup.conf"). JobAcctGatherParams=UsePSS NoShared proc . jobacct_gather/linux CPU procfs . pid /proc//stats . UsePSS /proc//smaps NoShare /proc//statm ( JobAcctGatherParams ). (spawned) (performance penalty) pid ppid . NOTE: . . JobAcctGatherFrequency . = . task . : : task= jobacct_gather acct_gather_profile. 30 . 0 () Slurm . : energy= acct_gather_energy. 0 () . network= InfiniBand acct_gather_interconnect. 0 () . filesystem= acct_gather_filesystem. 0 () . ( ) 30 10,000 . --acctg-freq . acct_gather_energy/ipmi xcc EnergyIPMIFreq (polls) . JobAcctGatherParams . : DisableGPUAcct GPU GPU . GPU . no_file_cache (active_file inactive_file) . memory.peak MaxRSS . cgroup/v2 . NoShared (shared memory) RSS . UsePSS . jobacct_gather/linux . OverMemoryKill JobAcctGather (kill) . / . NOTE: JobAcctGather task/cgroup TaskPlugin ConstrainRAMSpace=yes cgroup.conf . JobAcctGather (polling) (Out of Memory) . NOTE: OverMemoryKill JobAcctGather / (killed/cancelled) . ConstrainRAMSpace . UsePss RSS PSS . PSS RSS . NoShared . jobacct_gather/linux . JobCompHost . . UNIX "unix:/path/to/socket". JobCompLoc JobCompType : jobcomp/elasticsearch : (URL endpoint) Elasticsearch ( (index) ) . ://_doc . JobCompLoc . NOTE: . jobcomp/filetxt : . . /var/log/slurm_jobcomp.log . jobcomp/kafka : ( ) Kafka . librdkafka . key=value . bootstrap.servers librdkafka . JobCompLoc . NOTE: librdkafka . jobcomp_kafka : NOTE: () Kafka JobCompParams . jobcomp/lua : . jobcomp.lua slurm.conf . JobCompLoc . jobcomp/mysql : . . slurm_jobcomp_db . jobcomp/script : . . JobCompLoc . . JobCompParams . JobCompType. jobcomp/elasticsearch: send_comment (comment) jobcomp . send_script jobcomp . jobcomp/kafka: enable_job_start jobcomp/kafka topic_job_start . NOTE: ( ) . NOTE: . flush_timeout= ( ) (produce requests) . (timeout) API librdkafka . (producer instance) . (non- blocking) 0 . -1 ( fini slurmctld ). [-1,2147483647] . 500 () . poll_interval= poll API librdkafka (poll) . (thread) . [0,4294967295] . 2 () . requeue_on_msg_timeout (delivery report callback) message.timeout.ms librdkafka (requeue). ( ). send_comment (comment) jobcomp . send_script jobcomp . topic= . ClusterName . topic_job_start= . -job-start . NOTE: . jobcomp/mysql: token_duration StoragePassScript. . 300 (5 ) . JobCompPass . . JobCompPassScript (ephemeral) JobCompPass . / (stdout) 0 . . SlurmUser . : SLURM_STORAGE_HOSTNAME SLURM_STORAGE_PORT SLURM_STORAGE_USER : TOKEN= 0 . (stderr) . (backup host) . JobCompPort . . JobCompType () . (unset) . : jobcomp/elasticsearch Elasticsearch JobCompLoc . NOTE: Slurm ( https://slurm.schedmd.com/elasticsearch.html ). jobcomp/filetxt JobCompLoc . jobcomp/kafka ( ) Kafka JobCompLoc / JobCompParams . jobcomp/lua jobcomp.lua ( etc ) . jobcomp/mysql MySQL MariaDB JobCompLoc . jobcomp/script JobCompLoc . JobCompUser . . JobFileAppend . JobFileAppend 1 (append) . / (truncate) . JobRequeue (batch jobs) (requeue) . (preemption) . JobRequeue 1 . JobRequeue 0 . --no-requeue --requeue sbatch . 1 . JobSubmitPlugins (site-specific) / (logging) . Slurm job_submit . e.g. for multiple job_submit plugin configuration: JobSubmitPlugins=lua,require_timelimit . . : all_partitions . defaults . logging ( ) . lua Lua job_submit . Lua . "job_submit.lua" ( "etc" ) . Lua Slurm contribs/lua . lua slurmctld (fatal) . Slurm . slurmctld Slurm . Warning: slurmctld . slurmctld . . partition . pbs PBS Slurm ( ). require_timelimit (timelimit). NOTE: Slurm "src/plugins/job_submit" "contribs/lua/job_submit*.lua" . KillOnBadExit 1 (task) (crash) (abort) -- -- (step) . 0 . -K --kill-on-bad-exit srun . KillWait SIGTERM SIGKILL . (gracefully) . 30 . 65533 . MaxBatchRequeue (batch job) JobHeldAdmin . ( nohold_on_prolog_fail SchedulerParameters .) 5 . NodeFeaturesPlugins (node features) BIOS . active_features available_features . : node_features/helpers . helpers.conf : https://slurm.schedmd.com/helpers.conf.html LaunchParameters . : batch_step_set_cpu_freq CPU --cpu-freq CpuFreqDef slurm.conf. srun CPU . NOTE: srun ( ) cpu_freq srun . interactive_step_set_cpu_freq CPU --cpu-freq CpuFreqDef slurm.conf. cray_net_exclusive Cray XC . (oversubscribed). enable_nss_slurm passwd group slurmstepd (lookup) . https://slurm.schedmd.com/nss_slurm.html . lustre_no_flush Cray XC Lustre (flush) . . mpir_use_nodeaddr Slurm MPIR_proctable . MPIR_proctable MPIR_procdesc host_name NodeName . NodeAddr . disable_send_gids slurmctld (user_name) (extended gids) . . . enable_nss_slurm . slurmstepd_memlock slurmstepd RAM. slurmstepd_memlock_all slurmstepd RAM. srun_exclusive_allocation srun --exclusive --exclusive=allocation . --exact exclusive srun . srun --exclusive --exact . test_exec srun srun . use_interactive_step salloc salloc (Interactive Step) (shell) . srun InteractiveStepOptions . salloc . . ulimit_pam_adopt pam_slurm_adopt cgroup RLIMIT_RSS . LicenseParameters (licenses). . . : RemoteFuzzyMatch (remote licenses) . "matlab@licenseServer" "matlab" . . ( slurm.conf) . (fuzzy match) ( ) . Licenses ( ) . . ( "Licenses=foo:4,bar"). Slurm . Slurm . LogTimeFormat (timestamp) slurmctld slurmd. "iso8601" "iso8601_ms" "rfc5424" "rfc5424_ms" "rfc3339" "clock" "short" "thread_id" . "_ms" . "iso8601_ms" . "rfc5424" "iso8601" (timezone) . "clock" C ()clock . "short" . "thread_id" C ()ctime (PID) (thread) . MailDomain "--mail-user" . MTA . MailDomain . MailProg . "/bin/mail" ( "/usr/bin/mail" "/bin/mail" "/usr/bin/mail" ). mail . PrologSlurmctld EpilogSlurmctld : ALL SLURM_JOB_STATE MailProg . SLURM_JOB_MAIL_TYPE . BEGIN SLURM_JOB_QEUEUED_TIME . END, FAIL, REQUEUE, TIME_LIMIT_* SLURM_JOB_RUN_TIME . END, FAIL SLURM_JOB_EXIT_CODE_MAX (job array). SLURM_JOB_EXIT_CODE_MIN . SLURM_JOB_TERM_SIGNAL_MAX . STAGE_OUT SLURM_JOB_STAGE_OUT_TIME (staging out) . MaxArraySize MaxArraySize . MaxArraySize 0 . 4000001 . MaxJobCount MaxArraySize ( MaxJobCount ). 1001 . max_array_tasks SchedulerParameters . MaxDBDMsgs SlurmDBD slurmctld SlurmDBD . slurmctld . 10000 MaxJobCount * 2 + * 4 . 10000 . MaxJobCount slurmctld . MinJobAge slurmctld . . 10000 . NOTE: . . MaxSubmitJobs . Slurm . MaxJobId (job id) Slurm . . 67,108,863 (0x3FFFFFF) . 67,043,328 (0x03ff0000) . MaxJobId . . MaxJobId FirstJobId . 67,108,865 . FirstJobId . MaxMemPerCPU CPU (mebibytes). (over-subscribing) (paging) . MaxMemPerCPU (SelectType=select/cons_tres). 0 () . DefMemPerCPU DefMemPerGPU MaxMemPerNode . MaxMemPerCPU MaxMemPerNode . NOTE: CPU CPU . CPU . (best-effort) / QoS . LUA . MaxMemPerNode . . MaxMemPerNode (SelectType=select/linear) (OverSubscribe=yes OverSubscribe=force). 0 () . DefMemPerNode MaxMemPerCPU . MaxMemPerCPU MaxMemPerNode . MaxNodeCount . MaxNodeCount slurm.conf . MaxNodeCount slurm.conf . 65536 . scontrol MaxNodeCount . MaxStepCount . . 40000 . MaxTasksPerNode Slurm . MaxTasksPerNode 512 . 65533 . MCSParameters MCS = (Multi-Category Security). MCS. MCSPlugin . Slurm . MCS : . MCSPlugin MCS = (Multi-Category Security) : . . : mcs/account (account) ( ). mcs/group . mcs/user . mcs/label . MessageTimeout (round-trip) . 10 . slurmd (page out) . MetricsAuthUsers (endpoints) . SlurmUser root . PrivateData / MetricsParameters=ignore_private_data . JWT . MetricsParameters . . : ignore_private_data PrivateData . MetricsAuthUsers . MetricsType slurmctld . (Metrics Guide) : : metrics/openmetrics OpenMetrics. MinJobAge slurmctld . MaxJobCount slurmctld . 300 . . backfill backfill MinJobAge . (race conditions) MinJobAge 2 . MpiDefault MPI . Slurm MPI . srun . : pmi2 pmix. MPI : . MpiParams MPI. . : ports= PMI Cray . disable_slurm_hydra_bootstrap : I_MPI_HYDRA_BOOTSTRAP I_MPI_HYDRA_BOOTSTRAP_EXEC_EXTRA_ARGS HYDRA_BOOTSTRAP HYDRA_LAUNCHER_EXTRA_ARGS. I_MPI_HYDRA_BOOTSTRAP HYDRA_BOOTSTRAP 'slurm' . NamespaceType (namespaces) . 'PrologFlags=Contain' . : NOTE: ProctrackType . : namespace/linux (user) (pid) . (/tmp /dev/shm) . NOTE: cgroup/v2 . NOTE: bpf token ( 6.9) ConstrainDevices cgroup.conf . namespace/tmpfs (/tmp /dev/shm) . OverTimeLimit . (hard) . OverTimeLimit (soft) . OverTimeLimit . (backfill) . . 65533 . "UNLIMITED" . PluginDir Slurm . (:) PATH. "/lib/slurm" . PlugStackConfig (stackable) Slurm (SPANK) . / . "plugstack.conf" slurm.conf . SPANK spank(8) . PreemptMode (preempt) (gang scheduling). PreemptType PreemptMode . PreemptType=preempt/partition_prio PreemptMode . PreemptType=preempt/qos QOS . PreemptMode . GANG ( PreemptType) . PreemptMode ( PreemptMode=SUSPEND,GANG). . NOTE: backfill . backfill (preemptor) . backfill . backfill . NOTE: (heterogeneous job) . PreemptMode (SUSPEND () REQUEUE CANCEL ()) PreemptMode . GraceTime . GANG . OFF . (global) . . CANCEL . GANG ( time slicing) (suspended) . GANG . NOTE: GANG PreemptType=preempt/partition_prio PreemptParameters : reorder_count strict_order youngest_first. NOTE: OverSubscribe . PreemptType=preempt/partition_prio PriorityTier PriorityTier (Suspend) PreemptMode=SUSPEND,GANG Gang () . OverSubscribe FORCE . . NOTE: GANG . NOTE: : (Oversubscribe=FORCE Oversubscribe=YES -s/--oversubscribe ) . REQUEUE ( ) . sbatch --requeue JobRequeue slurm.conf 1 . SUSPEND Gang . SUSPEND GANG . Slurm . Slurm SIGTSTP PreemptParameters=suspend_grace_time ( ) SIGSTOP . SIGCONT . PreemptType=preempt/qos () Gang ( ). ( ) . NOTE: PreemptType=preempt/partition_prio PriorityTier PriorityTier . Gang . NOTE: GRES . GRES . PRIORITY . WITHIN PreemptType=preempt/qos qos . qos . PreemptParameters . min_exempt_priority= . . reclaim_licenses . . select/cons_tres . OR . reorder_count= . 1 . . strict_order . select/cons_tres . send_user_signal ( --signal=) . (gracetime preemption) SIGTERM . strict_order . . reorder_count . select/cons_tres . suspend_grace_time PreemptMode=SUSPEND . SIGTSTP SIGSTOP . 2 . NOTE: PreemptMode=SUSPEND scontrol suspend. GraceTime . youngest_first . ( preempt/partition_prio preempt/qos .) PreemptType . . preempt/partition_prio PriorityTier . PriorityTier PriorityTier . PreemptMode=OFF . preempt/qos (QOS) Slurm . PreemptMode=SUSPEND PriorityTier . QoS QoS (gang schedule) . PreemptMode=OFF . PreemptMode=SUSPEND SelectType=select/cons_tres . preempt/qos sacctmgr . PreemptExemptTime . PreemptExemptTime QOS . -1 0 . "minutes" "minutes:seconds" "hours:minutes:seconds" "days-hours" "days-hours:minutes" "days-hours:minutes:seconds" . PrEpParameters PrEpPlugins . PrEpPlugins Prolog Epilog (PrEp) . prep/script . . PrEp Plugin API : PriorityCalcPeriod (half-life decay) . PriorityType=priority/multifactor . 5 () . PriorityDecayHalfLife (association) ( ) . PriorityDecayHalfLife . 0 . . 0 PriorityUsageResetPeriod . PriorityType=priority/multifactor . ( min hr:min:00 days-hr:min:00 days-hr). 7-0 ( ) . PriorityFavorSmall . PriorityType=priority/multifactor . "YES" "NO" . "NO" . PriorityFlags . PriorityType=priority/multifactor . ( "PriorityFlags=ACCRUE_ALWAYS,SMALL_RELATIVE_TO_TIME"). ACCRUE_ALWAYS (holds) . (accrue limits) . CALCULATE_RUNNING . DEPTH_OBLIVIOUS . NO_FAIR_TREE . NO_FAIR_TREE "fair tree" "classic" . INCR_ONLY . . MAX_TRES TRES (TRESBillingWeights) : - (MAX) TRES ( cpus mem gres) - (SUM) TRES ( licenses) TRES : MAX_TRES_GRES TRES (TRESBillingWeights) : - (SUM) GRES ( GPU) - (MAX) TRES ( cpus mem) - (SUM) TRES ( licenses) Slurm 25.05 . TRES : NO_NORMAL_ALL NO_NORMAL_* . NO_NORMAL_ASSOC . NO_NORMAL_PART PriorityJobFactor . NO_NORMAL_QOS QOS qos . NO_NORMAL_TRES TRES TRES . SMALL_RELATIVE_TO_TIME . PriorityMaxAge . 30 30 . PriorityType=priority/multifactor . ( min hr:min:00 days-hr:min:00 days-hr). 7-0 ( ) . PriorityParameters PriorityType . PrioritySiteFactorParameters PrioritySiteFactorPlugin . PrioritySiteFactorPlugin "priority/multifactor" SiteFactor . . PriorityType . PriorityFlags . "priority/multifactor" . priority/basic (FIFO) . priority/multifactor (Fairshare) . FIFO : 1. 2. (advanced reservation) 3. PriorityTier 4. 5. 6. (Job ID) PriorityUsageResetPeriod 0 . . PriorityDecayHalfLife 0 . PriorityDecayHalfLife . PriorityType=priority/multifactor . NONE . . NOW . . DAILY . WEEKLY 00:00 . MONTHLY 00:00 . QUARTERLY 00:00 . YEARLY 00:00 . PriorityWeightAge . PriorityType=priority/multifactor . AccountingStorageType=accounting_storage/slurmdbd . 0 . PriorityWeightAssoc . PriorityType=priority/multifactor . 0 . PriorityWeightFairshare (fair-share) . PriorityType=priority/multifactor . AccountingStorageType=accounting_storage/slurmdbd . 0 . PriorityWeightJobSize . PriorityType=priority/multifactor . 0 . PriorityWeightPartition priority/multifactor . PriorityType=priority/multifactor . 0 . PriorityWeightQOS (Quality Of Service) . PriorityType=priority/multifactor . 0 . PriorityWeightTRES TRES TRES . e.g. PriorityWeightTRES=CPU=1000,Mem=2000,GRES/gpu=3000 PriorityType=priority/multifactor AccountingStorageTRES TRES . . 0 . PrivateData . . SlurmUser root . ( JWT ) SlurmUser root MetricsAuthUsers MetricsParameters=ignore_private_data . . : accounts ( SlurmDBD) (coordinator) . events (operator) . jobs . ( SlurmDBD) sacct . nodes . partitions . reservations . NOTE: QOS AllowedPartitions . usage 'sshare' 'scontrol show assoc_mgr' . ( SlurmDBD) sreport . users ( SlurmDBD) . . ProctrackType (job step) . slurmd . NOTE: "proctrack/linuxproc" "proctrack/pgid" init ( ) . "proctrack/cgroup" . NOTE: NamespaceType ProctrackType . : proctrack/cgroup cgroup cgroup . NOTE: "man cgroup.conf" . proctrack/linuxproc (parent process IDs) . proctrack/pgid (Process Group IDs) . NOTE: BSD . Prolog slurmd . ( ) slurm.conf . glob ( glob (7) ) ( "/etc/slurm/prolog.d/*"). prolog (z-a -> Z-A -> 9-0) . slurmd prolog . prolog . prolog . ( MessageTimeout) . prolog ( ) DRAIN (requeue) . (held) nohold_on_prolog_fail SchedulerParameters . Prolog and Epilog Scripts . NOTE: prolog . PrologEpilogTimeout Slurm Prolog Epilog . . Prolog Epilog slurmd PrologSlurmctld EpilogSlurmctld slurmctld prolog/epilog SPANK slurm_spank_job_prolog slurm_spank_job_epilog . PrologSlurmctld . Prolog slurm_spank_job_prolog (drain) . Epilog slurm_spank_job_epilog . . NOTE: PrologTimeout prolog . EpilogTimout epilog . PrologTimeout Slurm Prolog . PrologEpilogTimeout . Prolog slurmd PrologSlurmctld slurmctld prolog SPANK slurm_spank_job_prolog . PrologSlurmctld . Prolog slurm_spank_job_prolog (drain) . . PrologFlags Prolog. . . : Alloc Prolog . Prolog ( LaunchParameters=use_interactive_step ). Alloc . Cray . NOTE: Alloc . Contain ProcTrack (job container) . Slurm pam_slurm_adopt . pam_slurm_adopt ProcTrackType proctrack/cgroup . Contain Alloc . DeferBatch slurmctld prolog . Alloc slurmctld prolog . DeferBatch Alloc . NoHold salloc prolog . slurmd . srun . Contain X11 . NoHold Alloc . ForceRequeueOnFail Prolog . ForceRequeueOnFail Alloc . RunInJob Prolog/Epilog slurmstepd . . cgroup . RunInJob Contain Alloc . Serial Prolog Epilog . (job throughput) . NOTE: RunInJob . X11 X11 Slurm . ProctrackType=proctrack/linuxproc . X11 Contain Alloc . PrologSlurmctld slurmctld ( "/usr/local/slurm/prolog_controller"). SlurmUser slurmctld (drain) . . . POWER_UP/CONFIGURING . slurmctld . . ( scontrol) . ( ) . . Prolog and Epilog Scripts . NOTE: prolog . PropagatePrioProcess (nice value) . 0 slurm . . 1 ( srun sbatch) . root slurm . 2 ( srun sbatch) nice slurm ( slurm ). PropagateResourceLimits . slurmd ( soft) . . . . "--propagate" sbatch srun . PropagateResourceLimits PropagateResourceLimitsExcept "--propagate" . PropagateResourceLimits PropagateResourceLimitsExcept . (hard limits) slurmd . slurmd . Slurm /etc/sysconf/slurm . : https://slurm.schedmd.com/faq.html#memlock Slurm ( ): ALL () NONE AS ( ) . CORE core CPU CPU DATA (data segment) FSIZE . FSIZE slurmd.log 'File size limit exceeded' . MEMLOCK NOFILE NPROC RSS (resident set size). 2.4.30 BSD . STACK PropagateResourceLimitsExcept . ( PropagateResourceLimits ) . <<--propagate>> sbatch srun . PropagateResourceLimits . RebootProgram . <> <<--reboot>> . . PowerAction . ResumeTimeout . ResumeTimeout DOWN . ReconfigFlags <> . : KeepPartInfo <> (<>) <> . slurm.conf . KeepPartState . KeepPartState <> <> <> slurm.conf . slurm.conf . KeepPowerSaveSettings <> SuspendExcNodes SuspendExcParts SuspendExcStates . <> slurm.conf . RequeueExit (batch) . / <<->> ( "RequeueExit=1-9,18"). (pending) . SLURM_RESTART_COUNT . RequeueExitHold (held) . / <<->> ( "RequeueExitHold=10-12,16"). JOB_SPECIAL_EXIT . SLURM_RESTART_COUNT . ResumeFailProgram ResumeTimeout . ( hostlist Slurm). PowerAction . ResumeTimeout SuspendTimeout ( ) (kill) . ResumeProgram (Slurm) . . ResumeProgram . PowerAction . slurmctld ResumeProgram . ResumeProgram slurmd BootTime DOWN - ResumeTimeout . SchedulerParameters=requeue_on_resume_failure . ( multiple-slurmd ) slurmd <<-b>> . SlurmUser . ( hostlist Slurm). SLURM_RESUME_FILE JSON . slurmctld . ResumeProgram slurmctld ResumeProgram . ResumeProgram . . ResumeTimeout SuspendTimeout ( ) (kill) . ResumeRate ResumeProgram . ( ) . . . ResumeTimeout ( ) . DOWN . "Node unexpectedly rebooted." DOWN . INFINITE . ReturnToService . ResvEpilog slurmctld . . . . (epilog) . ResvOverRun . 0 ( ). "UNLIMITED" . ResvProlog slurmctld . . . (prolog) . ReturnToService DOWN . 0 . : 0 DOWN ( slurmd ). 1 DOWN DOWN . ( ) DOWN . GRES CPU slurm.conf . 2 DOWN . DOWN . GRES CPU slurm.conf . SchedulerParameters SchedulerType . . allow_zero_lic . assoc_limit_stop (association limits) . . batch_sched_delay= ( ) . ( sbatch) . . bb_array_stage_cnt= (burst buffer) . . 10 . bf_allow_magnetic_slot (backfill) bf . backfill bf . backfill bf_max_job_test . NOTE: . backfill ( ) backfill . NOTE: bf_one_resv_per_job bf . . bf_busy_nodes ( ) . . select/cons_tres . bf_continue . backfill . bf_hetjob_immediate . backfill . / QOS . . bf_hetjob_prio=min . bf_hetjob_prio=[min|avg|max] (backfill) PriorityType . ( ). ( ) ( ). . PriorityTier Priority priority/multifactor . (advanced reservation) ( ). Priority sprio . : backfill ( PriorityTier Priority ) . ( ) . . . bf_interval= . . SchedulerType=sched/backfill . : 30 : 1 : 10800 ( ). -1 backfill . bf_job_part_count_reserve= . bf_job_part_count_reserve=10 backfill . backfill . . bf_min_age_reserve bf_min_prio_reserve . : 0 : 0 : 100000. bf_licenses (license) . . bf_running_job_reserve . bf_max_job_array_resv= . . . . : 20 : 0 : 1000. NOTE: . bf_max_job_array_resv . bf_max_job_assoc= . bf_max_job_user . backfill . SchedulerType=sched/backfill . bf_max_job_user bf_max_job_part bf_max_job_test bf_max_job_user_part . bf_max_job_test bf_max_job_assoc . : 0 ( ) : 0 : bf_max_job_test. bf_max_job_part= . . SchedulerType=sched/backfill . partition_job_depth bf_max_job_test . bf_max_job_test bf_max_job_part . : 0 ( ) : 0 : bf_max_job_test. bf_max_job_start= . SchedulerType=sched/backfill . : 0 ( ) : 0 : 10000. bf_max_job_test= ( ). . backfill . . SchedulerType=sched/backfill . : 500 : 1 : 1,000,000. bf_max_job_user= . backfill . MAXIJOB Maui . SchedulerType=sched/backfill . bf_max_job_part bf_max_job_test bf_max_job_user_part . bf_max_job_test bf_max_job_user . : 0 ( ) : 0 : bf_max_job_test. bf_max_job_user_part= . SchedulerType=sched/backfill . bf_max_job_part bf_max_job_test bf_max_job_user . : 0 ( ) : 0 : bf_max_job_test. bf_max_time= ( ) ( ) . SchedulerType=sched/backfill . bf_interval ( ). : bf_interval ( ) : 1 : 3600 ( ). NOTE: bf_interval bf_max_time RPC . max_rpc_cnt . max_rpc_cnt ( RPC) . () . NOTE: backfill Slurm . max_rpc_cnt RPC . bf_min_age_reserve= . . . . bf_job_part_count_reserve bf_min_prio_reserve . : 0 : 0 : 2592000 ( ). bf_min_prio_reserve= . . . . bf_job_part_count_reserve bf_min_age_reserve . : 0 : 0 : 2^63. bf_node_space_size= node_space . backfill node_space . . SchedulerType=sched/backfill . bf_max_job_test bf_running_job_reserve . : bf_max_job_test : 2 : 2,000,000. bf_one_resv_per_job . - . . backfill - . ( hetjob) . backfill . - backfill . . ( ). . bf_resolution= . . . SchedulerType=sched/backfill . : 60 : 1 : 3600 ( ). bf_running_job_reserve backfill . . bf_topopt_enable backfill . src/plugins/sched/backfill/oracle.c . backfill ()oracle . SchedulerParameters=sched_interval=-1 . SchedulerParameters=bf_running_job_reserve . bf_topopt_iterations backfill . bf_topopt_enable . bf_window= . . . bf_window bf_resolution . SchedulerType=sched/backfill . : 1440 ( ) : 1 : 43200 ( ). bf_window_linear= . . . bf_window_linear . . 60 ( ) . ( ) . . . bf_window_linear . bf_yield_interval= . . bf_continue . bf_yield_sleep . : 2,000,000 ( ) : 1 : 10,000,000 ( ). bf_yield_rpc_cnt slurmctld . . Slurm max_rpc_cnt . max_rpc_cnt Slurm . : MAX((max_rpc_cnt / 10), 20) ( ) : 0 : 200. NOTE: max_rpc_cnt . . . bf_yield_sleep= . . bf_yield_interval . : 500,000 ( ) : 1 : 10,000,000 ( ). build_queue_timeout= . . 2,000,000 ( ) . correspond_after_task_cnt= aftercorr . . : 10. default_queue_depth= ( ) defer sched_min_interval . sched_interval ( ) . 100 . partition_job_depth . defer . ( ) . default_queue_depth . defer_batch defer . salloc/srun . delay_boot= . . . --delay-boot . disable_job_shrink . ( --no-kill .) disable_hetjob_steps . enable_hetjob_steps . . enable_user_top <> . extra_constraints --extra salloc sbatch srun Extra . ignore_constraint_validation --constraint . . . . ignore_prefer_validation . Ignore_NUMA ( AMD Opteron 6000) NUMA . Slurm (PU/ ) . Slurm NUMA . Ignore_NUMA NUMA . NOTE: hwloc 2.0 NUMA /CPU Slurm hwloc 2.0 Slurm HWLOC_OBJ_PACKAGE Socket SlurmdParameters=l3cache_as_socket . ignore_prefer_validation --prefer . . . ignore_constraint_validation . max_array_tasks . MaxArraySize . max_array_tasks=1000 MaxArraySize=100001 100000 1000 . max_rpc_cnt= slurmctld . . Slurm . : 0 ( ) : 0 : 1000. NOTE: (MAX_SERVER_THREADS) RPC . max_rpc_cnt backfill ( ) MAX(max_rpc_cnt/10, 20) RPC max_rpc_cnt=1000 RPC . . . max_sched_time= ( ) . Slurm RPC . MessageTimeout . MessageTimeout . . NOTE: . 'None' . max_script_size= . . . max_submit_line_size= . . . max_switch_wait= . . no_backup_scheduling . . Cray Cray . nohold_on_prolog_fail Prolog (held) . . pack_serial_at_end select/cons_tres . . partition_job_depth= Slurm / ( ). . bf_max_job_part . 0 ( ) . default_queue_depth . bf_max_job_part . reduce_completing_frag COMPLETING . CompleteWait COMPLETING . COMPLETING CompleteWait . . CompleteWait . NOTE: CompleteWait . CompleteWait=0 . NOTE: reduce_completing_frag . requeue_delay= . AuthInfo=cred_expire . requeue_on_resume_failure ResumeTimeout -- . PrologFlags=ForceRequeueOnFail . salloc_wait_nodes salloc ( ) . salloc . salloc --wait-all-nodes . sbatch_wait_nodes sbatch ( ) . sbatch . sbatch --wait-all-nodes . sched_interval= ( ) partition_job_depth . . -1 . sched_max_job_start= . . sched_min_interval= ( ) . ( ) . . . . . spec_cores_first . . step_retry_count= . step_retry_time . . step_retry_time= . step_retry_count . . time_min_as_soft_limit --time-min . (<<>>) --time . whole_hetjob . NOTE: whole_pack . SchedulerTimeSlice (time slice) (gang scheduling) (PreemptMode=SUSPEND,GANG). 5 65533 . 30 . SchedulerType . scontrol (jobs) . : sched/backfill (backfill) FIFO. (backfill) . backfill (time limits) (backfilling) . SchedulerParameters . . sched/builtin FIFO . . ( ) down/drained . . (heterogeneous jobs) backfill . ScronParameters . enable scrontab . explicit_scancel scrontab --cron scancel . SelectType (resource selection) . ( ) . cons_res (legacy) cons_tres . : select/cons_tres ( GPU ) (consumable resources) . Exclusive ( Exclusive=NODE) . Exclusive OverSubscribe . . select/linear . ( CPU ) TopologyPlugin=topology/flat CPU CPU . select/linear . linear (generic resources) . ( GPU) cons_tres . SelectTypeParameters SelectType . . SelectType . SelectType=select/linear . select/linear CR_ONE_TASK_PER_CORE ( ) CR_Memory (oversubscription) (job preemption) (gang scheduling) . SelectType=select/cons_tres CR_Core_Memory . select/cons_tres CPU CPU : CR_CPU (CPU) . CPUs hyper-thread . Boards Sockets CoresPerSocket ThreadsPerCore (locality) . CR_CPU_Memory (CPU) . CPU CR_CPU . DefMemPerCPU . CR_Core . hyper-threads (thread) CPU . CPU CPU --mem-per-cpu . CR_Core_Memory . CR_Core . DefMemPerCPU . CR_Socket . CPU . CR_Socket_Memory . CR_Socket . DefMemPerCPU . select/cons_tres : CR_ONE_TASK_PER_CORE . ThreadsPerCore (thread) . NOTE: CR_CPU* . CR_CORE_DEFAULT_DIST_BLOCK (block distribution) . - (pseudo-best-fit) ( ) . --distribution . (cyclically) . CR_LLN ( CPU ) . (serial jobs) (fragmentation) . Weight . LLN . CR_Pack_Nodes ( ) (pack) . CPU . . . --distribution=block . NoPack . LL_SHARED_GRES GRES (gres/mps gres/shard) ( ). GRES . MULTIPLE_SHARING_GRES_PJ GRES GRES . GRES GRES . : GPU 10 (shard) 12 2 (GPU) : 10 2 . ENFORCE_BINDING_GRES --gres-flags=enforce-binding . --gres-flags=disable-binding (override) . ONE_TASK_PER_SHARING_GRES --gres-flags=one-task-per-sharing . --gres-flags=multiple-tasks-per-sharing . CR_NO_DIST_TOPO_BLOCK (topology-aware) (steps). . (bitmap order) . NOTE: (CR_CPU CR_Core CR_Socket _Memory) (oversubscribe) cgroup.conf task/cgroup . --mem . . SlurmctldAddr slurmctld IP (Virtual IP) . SlurmctldHost . SlurmctldHost slurmctld . SlurmctldPrimaryOffProg SlurmctldPrimaryOnProg IP . SlurmctldDebug (logs) slurmctld. info . slurmctld -v --verbose (debug level) . quiet fatal (fatal) error info verbose debug debug2 debug3 debug4 debug5 SlurmctldHost Slurm ( "hostname -s" ). IP : SlurmctldHost=slurmctl-primary(12.34.56.78) slurmctld SlurmctldHost= : SlurmctldHost=slurmctl-primary1 SlurmctldHost=slurmctl-primary2 SlurmctldHost=slurmctl-primary3(12.34.56.78) SlurmctldHost . (primary) (standby backups) . standby primary . . Slurm ( "scontrol reconfig"). SlurmctldHost . slurmctld StateSaveLocation . RELOCATING CONTROLLERS . SlurmctldHttpAuthParameters HTTP slurmctld . JWT AuthAltParameters . . (endpoints) HTTP slurmctld AuthAltParameters . SlurmctldLogFile () slurmctld . none ( syslog ). LOGGING . SlurmctldParameters . allow_user_triggers () root/slurm_user . SlurmUser root . (man page) strigger . cloud_dns Slurm Slurm ( scontrol update nodename= nodeaddr=). Slurm slurm.conf Slurm IP . DNS . conmgr_connect_timeout= (timed out) . MessageTimeout . conmgr_max_connections= . (kernel) . (memory footprint) slurmctld . . 512 . conmgr_quiesce_timeout= (quiesce) . () . MessageTimeout . conmgr_read_timeout= (file descriptor) . MessageTimeout . conmgr_threads= (thread pool) . conmgr_threads . conmgr_use_poll poll(2) epoll(7) . conmgr_wait_write_delay= (poll) . MessageTimeout . conmgr_write_timeout= . MessageTimeout . disable_triggers . enable_async_reply slurmctld (asynchronous) RPC () conmgr. enable_configless " " (configless) slurmd slurmstepd . slurmd Prolog Epilog TaskProlog TaskEpilog slurmctld 'scontrol reconfigure' "configless" . https://slurm.schedmd.com/configless_slurm.html . NOTE: Include slurm.conf . NOTE: Prolog Epilog slurm.conf . (Glob patterns) ( glob (7) ) . enable_expedited_requeue (expedited requeue) . . enable_stepmgr slurmstepd . slurmstepd . . PrologFlags=contain . idle_on_node_suspend SuspendProgram (idle) (resumed) . max_dbd_msg_action MaxDBDMsgs 'discard' () 'exit' . 'discard' MaxDBDMsgs Step start Step complete MaxDBDMsgs Job start . Job completes MaxDBDMsgs (runaway jobs) . 'exit' MaxDBDMsgs slurmctld (exit ). slurmdbd slurmctld MaxDBDMsgs slurmctld . max_powered_nodes (powered up) . "scontrol power up " . node_reg_mem_percent= . 100 . State=CLOUD 90 . 100 . config_overrides . task/cgroup ConstrainRamSpace . cgroup . AllowedRamSpace cgroup.conf . no_quick_restart slurmctld (kill ). -i . power_save_interval power_save . power_save . 10 . power_save_min_interval power_save . 0 . reboot_from_controller RebootProgram slurmd. RebootProgram . RebootProgram PowerAction Location PowerAction . reconfig_on_restart slurmctld ( "scontrol reconfigure" ) slurmd sackd . "configless" . NOTE: (rolling upgrades) slurmd sackd . rl_bucket_size= (token bucket). (burst) RPC (steady-state) . 30 . rl_enable RPC (per-user rate-limiting). (back off) (sleep) . " " (token bucket) RPC RPC . rl_log_freq= ( ) RPC . 0 . -1 . 0 . rl_refill_period= ( ) . 1 . rl_refill_rate= . 2 . rl_table_size= (hash table) . . 8192 . user_resv_delete . validate_nodeaddr_threads= slurmctld . slurmctld . 1 . 64 . SlurmctldPidFile () slurmctld (PID) . . "/var/run/slurmctld.pid" . SlurmctldPort Slurm slurmctld . SLURMCTLD_PORT (build time) . 6817 . SlurmctldPort . NOTE: slurmctld slurmd SlurmctldPort SlurmdPort . NOTE: Cray IP (RSIP) 8192-60000 . SlurmctldPort SrunPortRange RSIP . SlurmctldPrimaryOffProg slurmctld . . . "SlurmctldPrimaryOnProg" . SlurmctldPrimaryOnProg slurmctld . . . IP (High Available) Slurm IP () ( slurmctld ARP ). "SlurmctldPrimaryOffProg" . SlurmctldSyslogDebug slurmctld syslog . slurmctld syslog fatal SlurmctldLogFile syslog SlurmctldDebug ( fatal SlurmctldDebug quiet ) quiet . quiet fatal (fatal) error info verbose debug debug2 debug3 debug4 debug5 NOTE: systemd Slurm slurmctld --systemd . systemd stdout/stderr Slurm syslog syslog . systemd "StandardOutput=null" "StandardError=null" override . SlurmctldTimeout ( ) . . . SlurmdDebug () slurmd. info . quiet fatal error info verbose debug debug2 debug3 debug4 debug5 SlurmdHttpAuthParameters HTTP slurmd . JWT AuthAltParameters . . HTTP slurmd AuthAltParameters (context) . SlurmdLogFile () slurmd . none ( syslog ). "%h" (hostname) slurmd . "%n" slurmd . LOGGING . SlurmdParameters Slurmd. . NOTE: Parameters . allow_ecores E-Core . ( E-Core .) config_overrides slurm.conf INVAL/INVALID_REG . . FastSchedule=2 . conmgr_connect_timeout= () . MessageTimeout . conmgr_max_connections= . (pending) . slurmd . . . conmgr_quiesce_timeout= (quiesce) . () . MessageTimeout . conmgr_read_timeout= (file descriptor) . MessageTimeout . conmgr_threads= (threads) . conmgr_threads . . conmgr_use_poll poll(2) epoll(7) . conmgr_wait_write_delay= (poll) . MessageTimeout . conmgr_write_timeout= . MessageTimeout . contain_spank namespace spank_user() spank_task_post_fork() spank_task_exit() (namespace) . l3cache_as_socket l3cache hwloc . l3cache ( CCX ). numa_node_as_socket . hwloc v2 . NOTE: slurm.conf (Sockets SocketsPerBoard CoresPerSocket ThreadsPerCore) slurmd -C --parameters=l3cache_as_socket . slurmd . numa_node_as_socket NUMA hwloc . NUMA . AMD Epyc . l3cache_as_socket . hwloc v2 . NOTE: slurm.conf (Sockets SocketsPerBoard CoresPerSocket ThreadsPerCore) slurmd -C --parameters=numa_node_as_socket . slurmd . shutdown_on_reboot Slurmd (reboot) . SlurmdPidFile slurmd (PID) . . "%h" slurmd . "%n" slurmd . "/var/run/slurmd.pid" . SlurmdPort slurmd . SLURMD_PORT . . NOTE: slurmctld slurmd SlurmctldPort SlurmdPort . NOTE: Cray Realm-Specific IP Addressing (RSIP) - . SlurmdPort SrunPortRange RSIP . SlurmdSpoolDir slurmd (batch job) . ( ). "/var/spool/slurmd" . "%h" slurmd . "%n" slurmd . SlurmdSyslogDebug slurmd syslog . slurmd fatal syslog SlurmdLogFile SlurmdDebug syslog ( SlurmdDebug quiet fatal) quiet . quiet fatal error info verbose debug debug2 debug3 debug4 debug5 NOTE: systemd slurmd --systemd . systemd stdout/stderr syslog syslog . systemd "StandardOutput=null" "StandardError=null" override . SlurmdTimeout ( ) DOWN slurmd . slurmctld slurmd DOWN slurmd slurmd . slurmd . . . SlurmdUser slurmd . . "root" slurmd . SlurmSchedLogFile (scheduling). SlurmctldLogFile . SlurmSchedLogFile SlurmSchedLogLevel . SlurmSchedLogLevel SlurmctldDebug slurmctld . SlurmSchedLogLevel "0" ( ) "1" ( ) . "0" () . SlurmSchedLogFile SlurmSchedLogLevel . scontrol . SlurmUser slurmctld . "root" . . "root" . SrunEpilog srun (job step) . . --epilog srun (override) . "Epilog" ( TaskEpilog) slurmd SrunEpilog "srun" . SrunPortRange srun slurmstepd / (I/O) . (ephemeral) . srun . . NOTE: Cray Realm-Specific IP Addressing (RSIP) - . SrunPortRange RSIP "SrunPortRange=60001-63000". NOTE: SrunPortRange srun . srun . --pty . : srun -N 1 . srun --pty -N 1 . srun -N 48 . srun -N 50 . srun -N 200 . SrunProlog srun . . --prolog srun (override) . "Prolog" ( TaskProlog) slurmd SrunProlog "srun" . StateSaveLocation slurmctld ( "/usr/local/slurm/checkpoint"). . SlurmUser . SlurmctldHost . ( RAID) . "/var/spool" . core dump . SuspendExcNodes (power save mode) (idle) . hostlist NodeSets ":" . "nid[10-20]:4" "nid[10-20]" . ( "nid[10-20]:4,nid[80-90]:2"). . scontrol . ReconfigFlags=KeepPowerSaveSettings . SuspendExcParts . . . scontrol . ReconfigFlags=KeepPowerSaveSettings . SuspendExcStates . CLOUD DOWN DRAIN DYNAMIC_FUTURE DYNAMIC_NORM FAIL INVALID_REG MAINTENANCE NOT_RESPONDING PLANNED RESERVED . SuspendTime . scontrol . ReconfigFlags=KeepPowerSaveSettings . SuspendProgram SuspendProgram . PowerAction . . . ( hostlist ). . ResumeTimeout SuspendTimeout (kill) . SuspendRate SuspendProgram . ( ) . . . SuspendTime (idle) / (down) SuspendProgram . (override) . SuspendTime INFINITE ( -1) . INFINITE . SuspendTimeout ( ) (suspend) . (resume) . . SwitchParameters (switch plugin). HPE Slingshot SwitchType=switch/hpe_slingshot ( ): vnis=- VNI . 1024-65535 . destroy_retries= (destroying) CXI . . . tcs=[:]... (traffic classes) . DEDICATED_ACCESS LOW_LATENCY BULK_DATA BEST_EFFORT. TC_DEDICATED_ACCESS TC_LOW_LATENCY TC_BULK_DATA TC_BEST_EFFORT . single_node_vni= 'all' VNI ( VNI ). 'user' VNI --network=single_node_vni srun SLURM_NETWORK=single_node_vni . 'none' ( single_node_vni ) VNI . single_node_vni 'all' . job_vni= 'all' VNI . 'user' VNI --network=job_vni srun SLURM_NETWORK=job_vni . 'none' ( job_vni ) VNI . job_vni 'all' . adjust_limits slurmd (NIC) ( ) . no_adjust_limits slurmd . . hwcoll_addrs_per_job (multicast) (hardware collectives) Slingshot ( hwcoll_min_nodes ). hwcoll_num_nodes Slingshot . fm_url URL fabric manager Slingshot . enable_stepmgr . fm_auth REST API HPE (BASIC OAUTH OAUTH). fm_authdir ( /etc/fmsim BASIC /etc/wlm-client-auth OAUTH). fm_authdir_ctld slurmctld . fm_authdir . fm_mtls_url URL fm_url mTLS fabric manager Slingshot . fm_url . mTLS fm_mtls_ca fm_mtls_cert fm_mtls_key . Note: fm_url enable_stepmgr Slingshot . fm_mtls_ca (CA bundle) fabric manager . Slingshot fabric manager . fm_mtls_cert fm_mtls_key . Note: mTLS fabric manager . ( slurmctld stepmgr) . fm_mtls_cert (public certificate) . mTLS fabric manager Slingshot . fm_mtls_ca fm_mtls_key . fm_mtls_key (private key) . mTLS fabric manager Slingshot . fm_mtls_ca fm_mtls_cert . nic_distribution_count= (NIC) . --network=nic_distribution_count= SLURM_NETWORK=nic_distribution_count= . NIC . def_= CPU . res_= . CPU . max_= . : txqs (Transmit command queues). CPU . tgqs (Target command queues). CPU . eqs (Event queues). CPU . cts (Counters). CPU . tles (Trigger list entries). CPU . ptes (Portable table entries). CPU . les (List entries). CPU . acs (Addressing contexts). CPU . SwitchType=switch/nvidia_imex : imex_channel_count . . . IMEX (batch) (interactive) . --network=unique-channel- per-segment topology/block IMEX . SwitchType (interconnect) . (Ethernet InfiniBand). Slurm SwitchType . slurmctld SwitchType . : switch/hpe_slingshot HPE Slingshot. switch/nvidia_imex NVIDIA IMEX. TaskEpilog (task) . ( ) slurm.conf . . . TaskProlog . TaskPlugin ( ). . "task/" . . : task/affinity ()sched_setaffinity (bind) . --cpu-bind / --mem-bind srun . task/cgroup cpuset Cgroups . --cpu-bind / --mem-bind srun . NOTE: "man cgroup.conf" . NOTE: TaskPlugin task/cgroup,task/affinity (stack) ConstrainCores=yes cgroup.conf . task/affinity cpu task/cgroup cpu . TaskPluginParam . . None Sockets Cores Threads --cpu-bind . CpuBind Node Partition . Cores . (override) . None . . Sockets . . Threads (threads) . . SlurmdSpecOverride slurmd cgroup cpuset CpuSpecList MemSpecLimit (override) . . cgroup/v1 slurmd slurmstepd cgroup/v1 CpuSpecList MemSpecLimit . SlurmdOffSpec CPU ( CoreSpecCount CpuSpecList ) Slurm ( slurmd slurmstepd) ( Slurm Slurm ). OOMKillStep OOM (kill) . (external step) . . NOTE: task/cgroup Cgroups v2 4.19 . Verbose . Autobind " " (auto binding) . Threads Cores Sockets ( TaskPluginParam=autobind=threads). TaskProlog . ( ) slurm.conf . . . SLURM_TASK_PID (PID) . . export NAME=value . . (exporting of functions) . print ... ( "print " ) . unset NAME . / (task prolog/epilog) : 1. pre_launch_priv() TaskPlugin 1. pre_launch() TaskPlugin 2. TaskProlog slurm.conf 3. User prolog --task-prolog srun SLURM_TASK_PROLOG 4. Task 5. User epilog --task-epilog srun SLURM_TASK_EPILOG 6. TaskEpilog slurm.conf 7. post_term() TaskPlugin TCPTimeout TCP. . TLSParameters TLS TLSType. : ca_cert_file= (CA). Slurm . 644 SlurmUser/root . "ca_cert.pem" Slurm ctld_cert_file= slurmctld. ca_cert_file . slurmctld . 600 SlurmUser . "ctld_cert.pem" Slurm ctld_cert_key_file= ctld_cert_file. slurmctld . 600 SlurmUser . "ctld_cert_key.pem" Slurm restd_cert_file= slurmrestd. ca_cert_file . slurmrestd . 600 slurmrestd . "restd_cert.pem" Slurm restd_cert_key_file= restd_cert_file. slurmrestd . 600 slurmrestd . "restd_cert_key.pem" Slurm sackd_cert_file= sackd. ca_cert_file . sackd . 600 SlurmUser . "sackd_cert.pem" Slurm NOTE: certmgr . sackd_cert_key_file= sackd_cert_file. sackd . 600 SlurmUser . "sackd_cert_key.pem" Slurm NOTE: certmgr . slurmd_cert_file= slurmd. ca_cert_file . slurmd . 600 SlurmUser . "slurmd_cert.pem" Slurm NOTE: certmgr . slurmd_cert_key_file= slurmd_cert_file. slurmd . 600 SlurmUser . "slurmd_cert_key.pem" Slurm NOTE: certmgr . load_system_certificates ( /etc/ssl) (trust store). ca_cert_file . security_policy_version= s2n. s2n . "20230317" FIPS TLS 1.3 . TLSType TLS . TLS RPC TLS . TLS : tls/none . : tls/none TLS. . tls/s2n TLS s2n. Slurm . TmpFS . TmpDisk . "/tmp" . TopologyParam . Dragonfly Dragonfly. TopologyPlugin=topology/tree . RoutePart . TreeWidth . . . BlockAsNodeRank (node rank) (base block). . topology/block . SwitchAsNodeRank (leaf switch). . topology/tree . RouteTree topology.conf . TopologyPlugin=topology/tree . (dynamic nodes). TopoMaxSizeUnroll= min-max nodes (unroll) . : -1 ( ). TopologyPlugin=topology/block . TopoOptional switch . . (fragment) . NOTE: . TopologyPlugin (network contention) . NETWORK TOPOLOGY . . : topology/block topology.conf(5) . topology/flat (best-fit) . . topology/tree select/cons_tres topology.conf(5) . NOTE: topology.yaml . TrackWCKey yes no. (Workload Characterization Key WCKey) . wckey . NOTE: TrackWCKey slurmdbd.conf . TreeWidth Slurmd . TreeWidth ( fanout) . 16 slurmd 16 slurmd . slurmctld ( 16 ) (4368 [message hops] ). . 65533 . UnkillableStepProgram UnkillableStepTimeout (unkillable) UnkillableStepProgram . . UNKILLABLE STEP PROGRAM SCRIPT . UnkillableStepTimeout Slurm ( SIGKILL ) UnkillableStepProgram . 60 MessageTimeout . (drain) . NOTE: UnkillableStepTimeout 5 MessageTimeout . UrlParserType url_parser . url_parser/libhttp_parser url_parser/internal. : url_parser/libhttp_parser libhttp_parser. url_parser/internal URL Slurm. Note: HttpParserType=http_parser/llhttp_parser . UsePAM 1 PAM ( ) . PAM . PAM . . 0 ( PAM). PAM Slurm . PAM slurm . module-type control-flags module-path : auth required pam_localuser.so auth required pam_shells.so account required pam_unix.so account required pam_access.so session required pam_unix.so PAM ( ) slurm (service-name) . . NOTE: UsePAM contribs/pam/pam_slurm / contribs/pam_slurm_adopt . UsePAM . VSizeFactor ( resident set size ) . . VSizeFactor . 500MiB VSizeFactor 101 500MiB 505MiB (101 ) (kill) . 0 . 65533 . NOTE: OverMemoryKill JobAcctGatherParams . TaskPlugin task/cgroup . VSizeFactor cgroups . WaitTime srun . "--wait" srun (override) . 0 . 65533 . X11Parameters (built-in) X11 Slurm. home_xauthority xauth TmpFS ~/.Xauthority . (NODE CONFIGURATION) ( ) Slurm /etc/slurm.conf . ( ) slurmctld slurmd . slurmd . NodeName . . (baseline node configurations) (heterogeneous) . ( ) "DOWN" . Slurm ( ) . : CPUs RealMemory TmpDisk. NodeName "DEFAULT" . "NodeName=DEFAULT" . NodeName "DEFAULT" . "NodeName=" . NodeName ( ). . . . . "DownNodes=" DOWN DRAIN FAILING . . Slurm . ( "alpha,beta,gamma") / . / "-" ( "linux[0-64,128]" "lx[15,18,32-33]" "rack[0-63]_blade[0-41]"). ( "linux[0000-1023]"). : NodeName Slurm . "/bin/hostname -s" . (FQDN) "/bin/hostname -f" ( "foo1.bar.com") (/etc/hosts) DNS (resolver). hostlist ( ). NodeHostname . NodeName "DEFAULT" . NodeName "DEFAULT" . . "NodeName=charlie" "NodeName=baker" . NOTE: NodeName "ALL" . NodeHostname "/bin/hostname -s" . "/bin/hostname -f" ( "foo1.bar.com") (/etc/hosts) DNS . hostlist ( ). . NodeHostname NodeName . NodeHostname NodeName . NodeAddr . ()getaddrinfo . NodeName ( "NodeName=lx[0-7] NodeAddr=elx[0-7]"). NodeAddr IP . NodeAddr NodeHostname . BcastAddr sbcast . ()getaddrinfo . NodeName ( "NodeName=lx[0-7] BcastAddr=elx[0-7]"). BcastAddr IP . BcastAddr sbcast NodeAddr . : CommunicationParameters=NoInAddrAny . Boards (Baseboards) . Boards SocketsPerBoard CoresPerSocket ThreadsPerCore . 1 . CoreSpecCount . SlurmdOffSpec TaskPluginParam Slurm slurmd ( ) . cgroup/v1 slurmstepd . slurmd . AllowSpecResourcesUsage=yes CoreSpecCount . CpuSpecList . Slurm ( https://slurm.schedmd.com/core_spec.html ) . CoresPerSocket ( "2"). CoresPerSocket . NOTE: . 1 . CpuBind (job step request) CPU ( --cpu-bind) CpuBind CpuBind . () TaskPluginParam (no binding) . CpuBind none socket ldom (NUMA) core thread. CPUs ( "2"). ( select/linear ) . hyper-threaded . CPUs Boards Sockets CoresPerSocket ThreadsPerCore . CpuSpecList CPU Slurm . CPU ( ) . SlurmdOffSpec TaskPluginParam Slurm slurmd () . cgroup/v1 slurmstepd . slurmd . AllowSpecResourcesUsage=yes CPU . CoreSpecCount . cgroup ( TaskPlugin task/cgroup ConstrainCores=yes cgroup.conf ). Features . . . --constraint . . Gres . GRES (Boolean) . NOTE: feature{myfeature} . slurmctld . : scontrol update node=feature{myfeature} state=resume PartitionName=p1 Nodes=feature{myfeature}. Gres . : "[:][:no_consume]:[K|M|G]". GresType . type . GRES GRES . no_consume . no_consume GRES GRES . GRES GPU GRES "shard" . . "K" "M" "G" "T" "P" . ( "Gres=gpu:tesla:1,gpu:kepler:1,bandwidth:lustre:no_consume:4G"). Gres (NodeName=DEFAULT) (Gres="") Gres . . (Boolean) Features . MemSpecLimit RealMemory . RealMemory . task/cgroup ( TaskPlugin task/cgroup ConstrainRAMSpace=yes cgroup.conf ) slurmd . cgroup/v1 slurmstepd . cgroup/v2 slurmstepd . Memory SelectTypeParameters . ( Out-Of-Memory Killer cgroup ). task/cgroup . Parameters SlurmdParameters . . Port Slurm slurmd . slurmd SlurmdPort . . slurmd . NOTE: Cray IP (RSIP) - . Port SrunPortRange RSIP . Procs CPUs . RealMemory ( "2048"). 1 . RealMemory Memory SelectTypeParameters . *_Memory . MemSpecLimit . Reason "DOWN" "DRAINED" "DRAINING" "FAIL" "FAILING" . . RestrictedCoresPerGPU GPU GPU . GPU . gres.conf(5) . NOTE: GPU GPU gpu . NOTE: gpu gres (shards mps) gres . Sockets / ( "2"). Sockets CPUs CoresPerSocket ThreadsPerCore . . Sockets SocketsPerBoard . Sockets Boards Sockets SocketsPerBoard . 1 . NOTE: SlurmdParameters=numa_node_as_socket SlurmdParameters=l3cache_as_socket Sockets CoresPerSocket ThreadsPerCore slurmd -C . SlurmdParameters . SocketsPerBoard / (baseboard). Sockets SocketsPerBoard . 1 . SuspendTime (idle) (down) SuspendProgram . SuspendTime . SuspendTime . . SuspendTime INFINITE . SuspendTime INFINITE ( -1) . State . CLOUD DOWN DRAIN FAIL FAILING FUTURE UNKNOWN. BUSY IDLE UNKNOWN . UNKNOWN BUSY IDLE . UNKNOWN . DownNodes . CLOUD (cloud) . (powered down) . Slurm slurmd . DOWN . DRAIN . FAIL . FAILING . FUTURE Slurm . scontrol slurmctld . State slurm.conf . Slurm . FUTURE -FUTURE . scontrol FUTURE . Dynamic Future Nodes slurmd -F[] FUTURE ( ) slurmd -C . NodeAddr NodeHostname slurmd FUTURE . UNKNOWN slurmd ( BUSY IDLE ). UNKNOWN . ThreadsPerCore ( "2"). Slurm . SelectTypeParameters=CR_CPU CPUs Sockets CoresPerSocket ThreadsPerCore . . select/cons_tres SelectTypeParameters CR_CPU . 1 . TmpDisk TmpFS ( "16384"). TmpFS ( "Temporary File System") . . . Prolog / Epilog ( ) . 0 . Topology :. . -- ':' -- ( Topology=topo- tree:sw_root:s1:s2). topology.conf(5) topology.yaml(5) . Weight . . . . . ( ). ( ) Weight . 1 . NOTE: . POWERED_DOWN POWERING_UP IDLE . (DOWN NODE CONFIGURATION) DownNodes= NodeName= DOWN DRAIN FAIL FAILING FUTURE . DownNodes NodeName=. Reason DOWN DRAIN FAIL FAILING FUTURE . . State . DOWN DRAIN FAIL FAILING FUTURE. State NodeName= . DOWN . (NODESET CONFIGURATION) (nodeset) (condo-style) . / . Feature= Nodes= . . Feature . . Nodes . NodeSet . NodeName . (PARTITION CONFIGURATION) ( ) . . ( ). . PartitionName "DEFAULT" . "PartitionName=DEFAULT" . "PartitionName=" . PartitionName "DEFAULT" . PartitionName ( ). slurm (scontrol reconfigure) . NOTE: . . : AllocNodes . . "ALL" . AllowAccounts . "ALL" . . NOTE: AllowAccounts DenyAccounts . DenyAccounts . AllowGroups . AllowGroups . root SlurmUser AllowGroups . Slurm AllowGroups . root ( --uid srun) AllowGroups . AllowGroups . 'ALL' . . . PrivateData . NOTE: Slurm . slurmctld ( "scontrol reconfig") AllowGroups ( "scontrol update PartitionName=name AllowGroups=group"). Slurm . AllowQos QOS . root AllowQos . "ALL" . NOTE: AllowQos DenyQos . DenyQos . Alternate "DRAIN" "INACTIVE" . CpuBind CPU ( --cpu-bind) CpuBind CpuBind . TaskPluginParam . CpuBind none socket ldom (NUMA) core thread. Default . "YES" "NO" . "NO" . DefaultTime . (MaxTime) QOS (MaxWall GrpWall) . MaxTime . MaxTime . DefCpuPerGPU CPU GPU . --cpus-per-task --cpus-per-gpu . DefMemPerCPU CPU . . DefMemPerCPU (SelectType=select/cons_tres). DefMemPerCPU . DefMemPerGPU DefMemPerNode MaxMemPerCPU . DefMemPerCPU DefMemPerGPU DefMemPerNode . DefMemPerGPU GPU . GPU MaxMemPerNode . DefMemPerCPU DefMemPerNode MaxMemPerCPU . DefMemPerCPU DefMemPerGPU DefMemPerNode . DefMemPerNode . . DefMemPerNode (SelectType=select/linear) (OverSubscribe=yes OverSubscribe=force). DefMemPerNode . DefMemPerCPU DefMemPerGPU MaxMemPerCPU . DefMemPerCPU DefMemPerGPU DefMemPerNode . DenyAccounts . . . NOTE: AllowAccounts DenyAccounts . AllowAccounts . DenyQos QOS . QOS . NOTE: AllowQos DenyQos . AllowQos . DisableRootJobs "YES" root . DisableRootJobs ( "NO" root ). Exclusive OverSubscribe . "NO" "NODE" "USER" "TOPO" . NODE OverSubscribe=NO . USER ( --exclusive=user). TOPO ( --exclusive=topo) OverSubscribe=NO . CPU --mem=0 ( ). OverSubscribe . OverSubscribe=EXCLUSIVE ExclusiveUser ExclusiveTopo . ExclusiveUser=YES ExclusiveTopo=YES . ExclusiveUser . GraceTime . PreemptType=partition_prio . . GraceTime . SIGCONT SIGTERM . SIGCONT SIGTERM SIGKILL . ( ) . KillWait . NOTE: PreemptMode=SUSPEND . PreemptMode=SUSPEND PreemptParameters=suspend_grace_time . NOTE: GraceTime PreemptMode ( PreemptMode=REQUEUE ). Slurm GraceTime . Hidden . API Slurm . "YES" "NO" . "NO" . AllowGroups . LLN ( CPU ) . . Weight . CR_LLN SelectTypeParameters . MaxCPUsPerNode CPU . GPU . Slurm ( "cpu" "gpu") / "cpu" CPU CPU / "gpu" . MaxCPUsPerSocket . MaxCPUsPerSocket CPU . GPU . MaxCPUsPerNode . MaxMemPerCPU CPU . . MaxMemPerCPU (SelectType=select/cons_tres). MaxMemPerCPU . DefMemPerCPU MaxMemPerNode . MaxMemPerCPU MaxMemPerNode . MaxMemPerNode . . MaxMemPerNode (SelectType=select/linear) (OverSubscribe=yes OverSubscribe=force). MaxMemPerNode . DefMemPerNode MaxMemPerCPU . MaxMemPerCPU MaxMemPerNode . MaxNodes . "UNLIMITED" -1 . MaxTime . : :: - -: -:: "UNLIMITED" . . TimeLimit root SlurmUser MaxTime . MinNodes . 0 . Nodes (nodesets) . . ( ) ( Nodes="") . "ALL" . OverSubscribe ( SelectTypeParameters) . (over-subscribed) . SelectTypeParameters --mem . (gang scheduling) (PreemptMode=suspend,gang) . OverSubscribe "FORCE" "YES" "NO". Exclusive=node ( ). "YES" "FORCE" . "NO" . : https://slurm.schedmd.com/cons_tres.html https://slurm.schedmd.com/cons_tres_share.html https://slurm.schedmd.com/gang_scheduling.html https://slurm.schedmd.com/preempt.html FORCE ( GRES) . . OverSubscribe=FORCE:4 . PreemptMode=suspend,gang . NOTE: OverSubscribe=FORCE:1 OverSubscribe=NO . OverSubscribe=FORCE:1 (preemption) PriorityTier . OverSubscribe=NO . NOTE: PreemptType=preempt/qos 1 FORCE . OverSubscribe=FORCE:2 QOS . NOTE: OverSubscribe slurm.conf FORCE YES (PreemptMode=OFF) (accounting) . slurmdbd : "We have more allocated time than is possible." YES ( GRES) . "--oversubscribe" . . "OverSubscribe=YES:4" . (PreemptMode=suspend,gang) . NO . . NOTE: PreemptMode=suspend,gang OverSubscribe=NO . OverSubscribe=FORCE:1 . OverTimeLimit . (hard) (killed) . OverTimeLimit (soft) . OverTimeLimit . backfill . OverTimeLimit . 65533 . "UNLIMITED" . PartitionName ( "Interactive"). . PartitionName "DEFAULT" . PartitionName "DEFAULT" . PowerDownOnIdle "YES" (power saving) (power down) . COMPLETING IDLE . "NO" . "NO" . . NOTE: COMPLETING IDLE : . Exclusive=USER . Exclusive=NODE . NOTE: (IDLE) SuspendTime PowerDownOnIdle NO . SuspendTime . PreemptMode PreemptType=preempt/partition_prio . PreemptMode PreemptMode (override) . OFF . PriorityTier PreemptMode . . NOTE: backfill . backfill (preemptor) . backfill . backfill . NOTE: (heterogeneous job) . PreemptMode (SUSPEND () REQUEUE CANCEL ()) PreemptMode . GraceTime . GANG . OFF . CANCEL . REQUEUE ( ) . sbatch --requeue JobRequeue slurm.conf 1 . SUSPEND Gang . SUSPEND GANG . Slurm . . NOTE: (gang scheduling) PreemptType=preempt/partition_prio PriorityTier PriorityTier . Gang . NOTE: GRES . GRES . PRIORITY . PriorityJobFactor priority/multifactor . 1 . 0 . 65533 . PriorityTier . PriorityTier PriorityTier PriorityTier . PreemptType=preempt/partition_prio () PriorityTier . 65533 . PriorityJobFactor . QOS QOS . QOS . QOS . QOS . QOS QOS . NOTE: QOS QOS QOS QOS OverPartQOS QOS . ReqResv (reservation) . . "YES" "NO" . "NO" . ResumeTimeout ( ) (resume) . DOWN (requeued). (reboot) DOWN "Node unexpectedly rebooted." . . ResumeTimeout . 65533 INFINITE . RootOnly ( root) SlurmUser . "YES" "NO" . "NO" . SelectTypeParameters . SelectTypeParameters . CR_Core CR_Core_Memory CR_Socket CR_Socket_Memory. SelectTypeParameters . State . "UP" "DOWN" "DRAIN" "INACTIVE". "UP" . "Alternate" . UP . DOWN . . . DRAIN ( ) . "Alternate" . INACTIVE . "Alternate" . SuspendTime (idle) (down) SuspendProgram (power save) . . SuspendTime . SuspendTime NodeName . SuspendTime INFINITE . SuspendTime INFINITE ( -1) . SuspendTimeout ( ) . (resume) . . SuspendTimeout . Topology topology.yaml . TRESBillingWeights = (billing weights) TRES ( AccountingStorageTRES ) . (fairshare) TRES . TRES PriorityWeightTRES . CPU (TRESBillingWeights="CPU=1"). TRES : (POWERACTION CONFIGURATION) PowerAction (reboot) (Resume) (Suspend) . PowerAction ResumeProgram ResumeFailProgram SuspendProgram RebootProgram "action" "scontrol power" "scontrol reboot" . PowerAction= Location= Program=" [arguments...]" : PowerAction . ResumeProgram ResumeFailProgram SuspendProgram RebootProgram "action" scontrol . Location "slurmd" "slurmctld" . "slurmctld" SlurmUser . "slurmd" SlurmdUser . NOTE: (Power up) "slurmctld" "slurmd" . Program . . Slurm () . Location=slurmd SLURM_NODE_NAME . PROLOG EPILOG (PROLOG AND EPILOG SCRIPTS) prolog epilog . : Prolog Epilog ( ) PrologSlurmctld EpilogSlurmctld ( SlurmctldHost ). NOTE: . . NOTE: Prolog (job step) . . Prolog . Prolog PrologFlags . Epilog . Epilog ( ) DRAIN . EpilogSlurmctld ( ) . Prolog ( ) DRAIN (requeued) . (held) nohold_on_prolog_fail SchedulerParameters . PrologSlurmctld ( ) . (batch jobs) . (salloc srun) PrologSlurmctld . slurmctld PrologSlurmctld EpilogSlurmctld SIGKILL . slurmctld . . Prolog Epilog . (UNKILLABLE STEP PROGRAM SCRIPT) (unkillable) / . SlurmdUser ( "root") UnkillableStepTimeout . . SLURM_JOB_ID (Job ID). SLURM_JOB_SLUID SLUID . SLURM_STEP_ID (Job Step ID). "batch" "interactive" "extern" 4294967291 4294967290 4294967292 . (NETWORK TOPOLOGY) (Slurm) (network contention) . Slurm (three- dimensional interconnect) : . Slurm . TopologyPlugin . "topology/flat" ( ) "topology/tree" "topology/block" ( topology.conf topology.yaml "man topology.conf" "man topology.yaml" ). . . Slurm (best-fit) . (RELOCATING CONTROLLERS) . : 1. Slurm . 2. slurm.conf . 3. StateSaveLocation . 4. slurm.conf . 5. Slurm . . . slurm.conf . CAUTION: ( SlurmctldHost slurmctld ) . SlurmctldHost . (EXAMPLE) # # /etc/slurm.conf dev[0-25].llnl.gov # Author: John Doe # Date: 11/06/2001 # SlurmctldHost=dev0(12.34.56.78) # Primary server SlurmctldHost=dev1(12.34.56.79) # Backup server # AuthType=auth/munge Epilog=/usr/local/slurm/epilog Prolog=/usr/local/slurm/prolog FirstJobId=65536 InactiveLimit=120 JobCompType=jobcomp/filetxt JobCompLoc=/var/log/slurm/jobcomp KillWait=30 MaxJobCount=10000 MinJobAge=300 PluginDir=/usr/local/lib:/usr/local/slurm/lib ReturnToService=0 SchedulerType=sched/backfill SlurmctldLogFile=/var/log/slurm/slurmctld.log SlurmdLogFile=/var/log/slurm/slurmd.log SlurmctldPort=7002 SlurmdPort=7003 SlurmdSpoolDir=/var/spool/slurmd.spool StateSaveLocation=/var/spool/slurm.state TmpFS=/tmp WaitTime=30 # # Node Configurations # NodeName=DEFAULT CPUs=2 RealMemory=2000 TmpDisk=64000 NodeName=DEFAULT State=UNKNOWN NodeName=dev[0-25] NodeAddr=edev[0-25] Weight=16 # Update records for specific DOWN nodes DownNodes=dev20 State=DOWN Reason="power,ETA=Dec25" # # Partition Configurations # PartitionName=DEFAULT MaxTime=30 MaxNodes=10 State=UP PartitionName=debug Nodes=dev[0-8,18-25] Default=YES PartitionName=batch Nodes=dev[9-17] MinNodes=4 PartitionName=long Nodes=dev[9-17] MaxTime=120 AllowGroups=admin INCLUDE (INCLUDE MODIFIERS) "include" (modifiers) . . ( ) slurm.conf . %c slurm.conf . EXAMPLE ClusterName=linux include /home/slurm/etc/%c_config # : # "include /home/slurm/etc/linux_config" (FILE AND DIRECTORY PERMISSIONS) : slurmctld SlurmUser . slurmd root . (login) . . Epilog root . . . EpilogSlurmctld SlurmUser . . . HealthCheckProgram root . . . JobCompLoc SlurmUser . . MailProg SlurmUser . . . PowerAction Location=slurmd Program SlurmdUser . Location=slurmctld Program SlurmUser . Prolog root . . . PrologSlurmctld SlurmUser . . . ResumeProgram SlurmUser . . slurm.conf . . SlurmctldLogFile SlurmUser . . SlurmctldPidFile root . SlurmUser . . SlurmdLogFile root . . SlurmdPidFile root . . SlurmdSpoolDir root . 755 . . SrunEpilog . . SrunProlog . . StateSaveLocation SlurmUser . . SuspendProgram SlurmUser . . TaskEpilog . . TaskProlog . . UnkillableStepProgram SlurmdUser . . (LOGGING) Slurm . (mount) (cold-starting - ) . / (uid) <> . <> <> . Slurm FILE AND DIRECTORY PERMISSIONS . logrotate . slurmdbd . logrotate . /etc/logrotate.d/slurm . logrotate . ## # Slurm Logrotate Configuration ## /var/log/slurm/*.log { compress missingok nocopytruncate nodelaycompress nomail notifempty noolddir rotate 5 sharedscripts size=5M create 640 slurm root postrotate pkill -x --signal SIGUSR2 slurmctld pkill -x --signal SIGUSR2 slurmd pkill -x --signal SIGUSR2 slurmdbd exit 0 endscript } (COPYING) Copyright (C) 2002-2007 The Regents of the University of California. Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). Copyright (C) 2008-2010 Lawrence Livermore National Security. Copyright (C) 2010-2022 SchedMD LLC. Slurm . . Slurm (GNU General Public License) (Free Software Foundation) ( ) / . Slurm (MERCHANTABILITY) (FITNESS FOR A PARTICULAR PURPOSE). GNU General Public License . (FILES) /etc/slurm.conf (SEE ALSO) cgroup.conf(5), getaddrinfo(3), getrlimit(2), gres.conf(5), group(5), hostname(1), scontrol(1), slurmctld(8), slurmd(8), slurmdbd(8), slurmdbd.conf(5), srun(1), spank(8), syslog(3), topology.conf(5) Slurm 26.05 slurm.conf(5)