candide: no node excludes, tile store on each job's own /scratch

profiles/candide/config.yaml

@@ -2,31 +2,29 @@
 # `SP_PROFILE=candide workflow/bin/sp run`. Mirrors profiles/nibi/config.yaml;
 # only the cluster facts differ, and each is stated where it is set.
 executor: slurm
+# Relative, so it joins the plain-spelled working directory the Snakefile sets
+# rather than resolving to /automnt/<disk>, absent on the node owning the disk.
+slurm-logdir: .snakemake/slurm_logs
 # candide is a shared ~25-node cluster: cap concurrent submissions well below
 # what nibi takes.
 jobs: 150
 default-resources:
   mem_mb: 2000
   runtime: 120                   # minutes
   slurm_account: cusers
-  # comp nodes have TmpDisk=0, so the tile_shape group (--tmp, see tile.smk's
-  # TILE_SLURM_EXTRA) lands on pscomp; everything else may use either.
   slurm_partition: "comp,pscomp"
-  # n09/n17/n36 follow sp_validation's candide exclusions; n23 has measured
-  # SLURM job failures (sp_validation commit 2b40c99); n34/n35 must not get
-  # ShapePipe jobs (n34's /tmp filled with orphaned tile stores).
-  slurm_extra: "'--exclude=n09,n17,n23,n34,n35,n36'"
 software-deployment-method: [apptainer]
-# tile.smk's /local/scratch (the tile store, ~5.6 GB per real-data tile) is
-# bound to a shared NFS directory. candide has no /local; node /tmp (~29 GB,
-# shared by every job on the node) must not be used; and the writable per-job
-# /scratch/$USER/$SLURM_JOB_ID cannot be named here (literal paths only, see
-# profiles/nibi/config.yaml; --cleanenv drops SLURM_JOB_ID). The source dir
-# must exist. Cost: ngmix reads the store over NFS. A run can move it with
-# tile_store_root (workflow/config.yaml). Stores left by killed jobs are only
-# swept after a day by a later tile job; check this dir after failed campaigns.
+# Everything but source-cache, whose path would be the launch's and may be
+# node-local; the Snakefile drops XDG_CACHE_HOME so jobs derive their own.
+shared-fs-usage:
+  - persistence
+  - input-output
+  - software-deployment
+  - sources
+  - storage-local-copies
+  - software-deployment-cache
 # PYTHONPATH names a checkout's src/; workflow/bin/sp rewrites the launch copy.
-apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/n17data/mkilbing/astro/repositories/github/shapepipe/src --bind /home,/automnt,/n17data,/n23data1,/n09data --bind /n23data1/mkilbing/shapepipe/tilestore:/local/scratch"
+apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/n17data/mkilbing/astro/repositories/github/shapepipe/src --bind /home,/automnt,/n17data,/n23data1,/n09data"
 latency-wait: 60                 # NFS: wait for outputs to appear after a job
 keep-going: true                 # a failed job poisons only its cone; siblings run on
 # Retry transient node failures, including measured process-slot exhaustion
@@ -39,16 +37,15 @@ show-failed-logs: true
 printshellcmds: true
 rerun-triggers: [mtime, params, code, software-env]
 # @sc [label:coupling] candide-group-slurm-extra
-# tile_shape's rule-level slurm_extra replaces the profile default. Set both
-# the --tmp floor and node exclusions identically on all four member rules:
-# a group's string resources must agree. Leave rule threads and attempt-scaled
-# memory intact.
+# tile.smk's --tmp floor selects nodes by Slurm's configured TmpDisk, which is 0
+# on nodes that do have /scratch. Clear it on all four tile_shape member rules:
+# a group's string resources must agree.
 set-resources:
   tile_vignets:
-    slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n34,n35,n36'"
+    slurm_extra: "''"
   tile_ngmix:
-    slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n34,n35,n36'"
+    slurm_extra: "''"
   tile_merge_cats:
-    slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n34,n35,n36'"
+    slurm_extra: "''"
   tile_make_cat:
-    slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n34,n35,n36'"
+    slurm_extra: "''"

tests/helpers/cluster.py

@@ -53,14 +53,14 @@ def srun(args, *, partition="comp", time_limit="00:30:00", cpus=4, extra=None):
     cpus : int
         ``--cpus-per-task``.
     extra : list of str, optional
-        Extra ``srun`` flags (e.g. ``["--exclude", "n09,n17,n36"]``).
+        Extra ``srun`` flags (e.g. ``["--nodelist", "n24"]``).
     """
     cmd = [
         "srun",
         f"--partition={partition}",
         f"--time={time_limit}",
         f"--cpus-per-task={cpus}",
-        *(extra or ["--exclude", "n09,n17,n36"]),
+        *(extra or []),
         *args,
     ]
     return subprocess.run(cmd, check=True, text=True, capture_output=True)

workflow/README.md

@@ -133,11 +133,19 @@ catalogue as a data campaign, written by the same `final_cat_merge` rule:
 `config/cfis_image_sims/final_cat.param` and the tile count as the file's
 `n_tiles` attribute (there is no `n_tiles_final.txt`).
 
-On candide, the tile store is not node-local: the profile binds a shared NFS
-directory to `/local/scratch`, never the node's `/tmp`. To use another directory,
-set `tile_store_root:` in the run config; `sp run` binds it to `/local/scratch`
-for that campaign. The store names carry a per-campaign hash, so all branches can
-share one root.
+The tile store holds one tile's vignets and WCS for the length of its `tile_shape`
+job, so it wants the fastest disk the job's node has, and `tile_store_root` in the
+`machines:` table names it. On candide that's `/scratch/%u/%j`, the directory Slurm
+creates on the node's own disk for each job and deletes when the job ends. Never
+use `/tmp` on candide: it is ~30 GB shared by everyone on the node.
+
+On candide, write paths as `/nXXdataN/...`, never `/automnt/nXXdataN/...`: the
+`/automnt` form does not exist on the node that owns that disk. The Snakefile spells
+Snakemake's working directory in the plain form, so jobs can run on the node holding
+their own campaign.
+
+Don't exclude candide nodes from the profile without a reproducible failure of a
+pinned job on that node.
 
 ## The container image
 

workflow/Snakefile

@@ -83,6 +83,26 @@ if _unresolved:
         f"$variable for machine={MACHINE!r}, "
         f"input_type={INPUT_TYPE!r}: {', '.join(_unresolved)}. Set them in "
         f"your run config (SP_RUN_CONFIG).")
+
+# candide: the working directory resolves to /automnt/<disk>, which is absent on
+# the node owning the disk, while /<disk> is reachable everywhere. The head process
+# derives the paths it hands to jobs from it (their working directory, Slurm log
+# and wait-for files), so spell it in the plain form before any of them is made.
+if MACHINE == "candide":
+    workflow._workdir_init = workflow._workdir_init.removeprefix("/automnt")
+    os.getcwd = lambda getcwd=os.getcwd: getcwd().removeprefix("/automnt")
+
+# A login shell's XDG_CACHE_HOME may name a node-local directory other nodes
+# cannot create, and jobs inherit the launch's environment.
+os.environ.pop("XDG_CACHE_HOME", None)
+# Every job parses this file on its node, inside its allocation, so a store on
+# per-job disk can be bound here.
+_tile_store = config.get("tile_store_root")
+if _tile_store and ("%j" not in _tile_store or "SLURM_JOB_ID" in os.environ):
+    _tile_store = (_tile_store.replace("%u", os.environ.get("USER", ""))
+                   .replace("%j", os.environ.get("SLURM_JOB_ID", "")))
+    os.makedirs(_tile_store, exist_ok=True)
+    os.environ["APPTAINER_BIND"] = f"{_tile_store}:/local/scratch"
 # The external masks are a data-only input: config_tile_Mc.ini's
 # MASK_EXT_PATHS reads $SP_INPUT_MASKS, and the sims Mc.ini carries no mask
 # block. Checked here rather than in run_config.REQUIRED because it depends on

workflow/bin/sp

@@ -176,31 +176,6 @@ if not n:
 f.write_text(text)
 PY
 
-  # Move the tile store by rewriting the bind source, not tile_local()'s params:
-  # params changes invalidate finished tiles (workflow/CONTRACTS). This bind is
-  # not a rerun trigger. Append it if the profile has no X:/local/scratch bind.
-  # See workflow/config.yaml for tile_store_root's capacity/latency trade-off.
-  python - "$SNAPSHOT/profiles/$PROFILE/config.yaml" "$(cfg tile_store_root)" <<'PY'
-import os, pathlib, re, sys
-f, root = pathlib.Path(sys.argv[1]), sys.argv[2]
-if root:
-    root = str(root)
-    if not os.path.isabs(root):
-        sys.exit(f"sp: tile_store_root must be an absolute path, got {root!r}")
-    os.makedirs(root, exist_ok=True)
-    bind = f"--bind {root}:/local/scratch"
-    text, n = re.subn(r'--bind \S+:/local/scratch(?=[\s"])', lambda m: bind,
-                      f.read_text(), count=1)
-    if not n:
-        text, n = re.subn(r'(apptainer-args:\s*"[^"]*)"',
-                          lambda m: m.group(1) + " " + bind + '"', text, count=1)
-    if not n:
-        sys.exit("sp: tile_store_root is set but the profile has no quoted "
-                 "apptainer-args to bind it with")
-    f.write_text(text)
-    print(f"sp: tile store root {root} (bound to /local/scratch)", file=sys.stderr)
-PY
-
   python - "$SNAPSHOT/snapshot.json" "$REPO" <<'PY'
 import json, pathlib, subprocess, sys, time
 out, repo = pathlib.Path(sys.argv[1]), sys.argv[2]

workflow/config.yaml

@@ -98,12 +98,14 @@ machines:
     base_dir: /n17data/UNIONS/WL
     data:
       retrieve: vos
+      tile_store_root: /scratch/%u/%j
       inputs:
         masks: $base_dir/masks/dr6-2026-08
         catalogues: vos:cfis/tiles_DR6
       container: /n17data/cdaley/containers/shapepipe_develop-runtime-20260718.sif
     image_sims:
       retrieve: symlink
+      tile_store_root: /scratch/%u/%j
       inputs:
         # $run is the branch dir (e.g. 1p2z_grid_1), so a run config
         # picks the grid by setting `run:` alone.
@@ -225,6 +227,8 @@ ngmix_chunks: 8
 # interpolate.
 # defect_fill: noise
 
-# Tile store (vignets, WCS). Unset: the profile's /local/scratch bind (nibi
-# node NVMe; candide a shared NFS directory, see profiles/candide/config.yaml).
-# tile_store_root: /n23data1/<user>/shapepipe/tilestore
+# Tile store (vignets, WCS): intermediates of one tile_shape job, deleted when it
+# ends. tile_store_root is the directory bound to /local/scratch in that job, set
+# per machine in machines:; %u and %j are the user and Slurm job id. candide uses
+# Slurm's per-job directory on each node's own disk. Unset (nibi): the profile's
+# /local bind. A literal directory also works, for example to compare disks.

workflow/rules/tile.smk

@@ -92,7 +92,7 @@ def tile_local(tile):
     Derive a literal path from the tile wildcard and LOCAL_TAG (the run-dir
     hash), so concurrent campaigns over the same tile do not share a store.
     See profiles/nibi/config.yaml's /local bind note for container path handling
-    and bin/sp's tile_store_root handling for the host-side bind.
+    and the Snakefile's tile_store_root handling for the per-job bind.
     On nibi the container's /tmp is RAM-backed, so these stores go on the
     /local/scratch bind instead.