diff --git a/profiles/candide/config.yaml b/profiles/candide/config.yaml index 2cfe8d091..323aa16ef 100644 --- a/profiles/candide/config.yaml +++ b/profiles/candide/config.yaml @@ -2,6 +2,9 @@ # `SP_PROFILE=candide workflow/bin/sp run`. Mirrors profiles/nibi/config.yaml; # only the cluster facts differ, and each is stated where it is set. executor: slurm +# Relative, so it joins the plain-spelled working directory the Snakefile sets +# rather than resolving to /automnt/, absent on the node owning the disk. +slurm-logdir: .snakemake/slurm_logs # candide is a shared ~25-node cluster: cap concurrent submissions well below # what nibi takes. jobs: 150 @@ -9,24 +12,19 @@ default-resources: mem_mb: 2000 runtime: 120 # minutes slurm_account: cusers - # comp nodes have TmpDisk=0, so the tile_shape group (--tmp, see tile.smk's - # TILE_SLURM_EXTRA) lands on pscomp; everything else may use either. slurm_partition: "comp,pscomp" - # n09/n17/n36 follow sp_validation's candide exclusions; n23 has measured - # SLURM job failures (sp_validation commit 2b40c99); n34/n35 must not get - # ShapePipe jobs (n34's /tmp filled with orphaned tile stores). - slurm_extra: "'--exclude=n09,n17,n23,n34,n35,n36'" software-deployment-method: [apptainer] -# tile.smk's /local/scratch (the tile store, ~5.6 GB per real-data tile) is -# bound to a shared NFS directory. candide has no /local; node /tmp (~29 GB, -# shared by every job on the node) must not be used; and the writable per-job -# /scratch/$USER/$SLURM_JOB_ID cannot be named here (literal paths only, see -# profiles/nibi/config.yaml; --cleanenv drops SLURM_JOB_ID). The source dir -# must exist. Cost: ngmix reads the store over NFS. A run can move it with -# tile_store_root (workflow/config.yaml). Stores left by killed jobs are only -# swept after a day by a later tile job; check this dir after failed campaigns. +# Everything but source-cache, whose path would be the launch's and may be +# node-local; the Snakefile drops XDG_CACHE_HOME so jobs derive their own. +shared-fs-usage: + - persistence + - input-output + - software-deployment + - sources + - storage-local-copies + - software-deployment-cache # PYTHONPATH names a checkout's src/; workflow/bin/sp rewrites the launch copy. -apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/n17data/mkilbing/astro/repositories/github/shapepipe/src --bind /home,/automnt,/n17data,/n23data1,/n09data --bind /n23data1/mkilbing/shapepipe/tilestore:/local/scratch" +apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/n17data/mkilbing/astro/repositories/github/shapepipe/src --bind /home,/automnt,/n17data,/n23data1,/n09data" latency-wait: 60 # NFS: wait for outputs to appear after a job keep-going: true # a failed job poisons only its cone; siblings run on # Retry transient node failures, including measured process-slot exhaustion @@ -39,16 +37,15 @@ show-failed-logs: true printshellcmds: true rerun-triggers: [mtime, params, code, software-env] # @sc [label:coupling] candide-group-slurm-extra -# tile_shape's rule-level slurm_extra replaces the profile default. Set both -# the --tmp floor and node exclusions identically on all four member rules: -# a group's string resources must agree. Leave rule threads and attempt-scaled -# memory intact. +# tile.smk's --tmp floor selects nodes by Slurm's configured TmpDisk, which is 0 +# on nodes that do have /scratch. Clear it on all four tile_shape member rules: +# a group's string resources must agree. set-resources: tile_vignets: - slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n34,n35,n36'" + slurm_extra: "''" tile_ngmix: - slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n34,n35,n36'" + slurm_extra: "''" tile_merge_cats: - slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n34,n35,n36'" + slurm_extra: "''" tile_make_cat: - slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n34,n35,n36'" + slurm_extra: "''" diff --git a/tests/helpers/cluster.py b/tests/helpers/cluster.py index 1a8addcc5..c770bf6c3 100644 --- a/tests/helpers/cluster.py +++ b/tests/helpers/cluster.py @@ -53,14 +53,14 @@ def srun(args, *, partition="comp", time_limit="00:30:00", cpus=4, extra=None): cpus : int ``--cpus-per-task``. extra : list of str, optional - Extra ``srun`` flags (e.g. ``["--exclude", "n09,n17,n36"]``). + Extra ``srun`` flags (e.g. ``["--nodelist", "n24"]``). """ cmd = [ "srun", f"--partition={partition}", f"--time={time_limit}", f"--cpus-per-task={cpus}", - *(extra or ["--exclude", "n09,n17,n36"]), + *(extra or []), *args, ] return subprocess.run(cmd, check=True, text=True, capture_output=True) diff --git a/workflow/README.md b/workflow/README.md index 294cce234..4168b89a0 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -133,11 +133,19 @@ catalogue as a data campaign, written by the same `final_cat_merge` rule: `config/cfis_image_sims/final_cat.param` and the tile count as the file's `n_tiles` attribute (there is no `n_tiles_final.txt`). -On candide, the tile store is not node-local: the profile binds a shared NFS -directory to `/local/scratch`, never the node's `/tmp`. To use another directory, -set `tile_store_root:` in the run config; `sp run` binds it to `/local/scratch` -for that campaign. The store names carry a per-campaign hash, so all branches can -share one root. +The tile store holds one tile's vignets and WCS for the length of its `tile_shape` +job, so it wants the fastest disk the job's node has, and `tile_store_root` in the +`machines:` table names it. On candide that's `/scratch/%u/%j`, the directory Slurm +creates on the node's own disk for each job and deletes when the job ends. Never +use `/tmp` on candide: it is ~30 GB shared by everyone on the node. + +On candide, write paths as `/nXXdataN/...`, never `/automnt/nXXdataN/...`: the +`/automnt` form does not exist on the node that owns that disk. The Snakefile spells +Snakemake's working directory in the plain form, so jobs can run on the node holding +their own campaign. + +Don't exclude candide nodes from the profile without a reproducible failure of a +pinned job on that node. ## The container image diff --git a/workflow/Snakefile b/workflow/Snakefile index bb7c32933..d22fe5e5b 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -83,6 +83,26 @@ if _unresolved: f"$variable for machine={MACHINE!r}, " f"input_type={INPUT_TYPE!r}: {', '.join(_unresolved)}. Set them in " f"your run config (SP_RUN_CONFIG).") + +# candide: the working directory resolves to /automnt/, which is absent on +# the node owning the disk, while / is reachable everywhere. The head process +# derives the paths it hands to jobs from it (their working directory, Slurm log +# and wait-for files), so spell it in the plain form before any of them is made. +if MACHINE == "candide": + workflow._workdir_init = workflow._workdir_init.removeprefix("/automnt") + os.getcwd = lambda getcwd=os.getcwd: getcwd().removeprefix("/automnt") + +# A login shell's XDG_CACHE_HOME may name a node-local directory other nodes +# cannot create, and jobs inherit the launch's environment. +os.environ.pop("XDG_CACHE_HOME", None) +# Every job parses this file on its node, inside its allocation, so a store on +# per-job disk can be bound here. +_tile_store = config.get("tile_store_root") +if _tile_store and ("%j" not in _tile_store or "SLURM_JOB_ID" in os.environ): + _tile_store = (_tile_store.replace("%u", os.environ.get("USER", "")) + .replace("%j", os.environ.get("SLURM_JOB_ID", ""))) + os.makedirs(_tile_store, exist_ok=True) + os.environ["APPTAINER_BIND"] = f"{_tile_store}:/local/scratch" # The external masks are a data-only input: config_tile_Mc.ini's # MASK_EXT_PATHS reads $SP_INPUT_MASKS, and the sims Mc.ini carries no mask # block. Checked here rather than in run_config.REQUIRED because it depends on diff --git a/workflow/bin/sp b/workflow/bin/sp index e75370849..99b213b36 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -176,31 +176,6 @@ if not n: f.write_text(text) PY - # Move the tile store by rewriting the bind source, not tile_local()'s params: - # params changes invalidate finished tiles (workflow/CONTRACTS). This bind is - # not a rerun trigger. Append it if the profile has no X:/local/scratch bind. - # See workflow/config.yaml for tile_store_root's capacity/latency trade-off. - python - "$SNAPSHOT/profiles/$PROFILE/config.yaml" "$(cfg tile_store_root)" <<'PY' -import os, pathlib, re, sys -f, root = pathlib.Path(sys.argv[1]), sys.argv[2] -if root: - root = str(root) - if not os.path.isabs(root): - sys.exit(f"sp: tile_store_root must be an absolute path, got {root!r}") - os.makedirs(root, exist_ok=True) - bind = f"--bind {root}:/local/scratch" - text, n = re.subn(r'--bind \S+:/local/scratch(?=[\s"])', lambda m: bind, - f.read_text(), count=1) - if not n: - text, n = re.subn(r'(apptainer-args:\s*"[^"]*)"', - lambda m: m.group(1) + " " + bind + '"', text, count=1) - if not n: - sys.exit("sp: tile_store_root is set but the profile has no quoted " - "apptainer-args to bind it with") - f.write_text(text) - print(f"sp: tile store root {root} (bound to /local/scratch)", file=sys.stderr) -PY - python - "$SNAPSHOT/snapshot.json" "$REPO" <<'PY' import json, pathlib, subprocess, sys, time out, repo = pathlib.Path(sys.argv[1]), sys.argv[2] diff --git a/workflow/config.yaml b/workflow/config.yaml index d8fa2ac45..3f52b637e 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -98,12 +98,14 @@ machines: base_dir: /n17data/UNIONS/WL data: retrieve: vos + tile_store_root: /scratch/%u/%j inputs: masks: $base_dir/masks/dr6-2026-08 catalogues: vos:cfis/tiles_DR6 container: /n17data/cdaley/containers/shapepipe_develop-runtime-20260718.sif image_sims: retrieve: symlink + tile_store_root: /scratch/%u/%j inputs: # $run is the branch dir (e.g. 1p2z_grid_1), so a run config # picks the grid by setting `run:` alone. @@ -225,6 +227,8 @@ ngmix_chunks: 8 # interpolate. # defect_fill: noise -# Tile store (vignets, WCS). Unset: the profile's /local/scratch bind (nibi -# node NVMe; candide a shared NFS directory, see profiles/candide/config.yaml). -# tile_store_root: /n23data1//shapepipe/tilestore +# Tile store (vignets, WCS): intermediates of one tile_shape job, deleted when it +# ends. tile_store_root is the directory bound to /local/scratch in that job, set +# per machine in machines:; %u and %j are the user and Slurm job id. candide uses +# Slurm's per-job directory on each node's own disk. Unset (nibi): the profile's +# /local bind. A literal directory also works, for example to compare disks. diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index b86246557..c5abd8e59 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -92,7 +92,7 @@ def tile_local(tile): Derive a literal path from the tile wildcard and LOCAL_TAG (the run-dir hash), so concurrent campaigns over the same tile do not share a store. See profiles/nibi/config.yaml's /local bind note for container path handling - and bin/sp's tile_store_root handling for the host-side bind. + and the Snakefile's tile_store_root handling for the per-job bind. On nibi the container's /tmp is RAM-backed, so these stores go on the /local/scratch bind instead.