From 9daedededbaebdf92ca60c57e877473509b1545b Mon Sep 17 00:00:00 2001 From: Cail McLean Daley Date: Wed, 4 Nov 2026 12:22:40 +0100 Subject: [PATCH 1/7] candide profile: exclude only n17, and for its actual reason Removes the exclusions of n09, n23, n34, n35 and n36, and the --tmp floor, which only kept tile_shape off the TmpDisk=0 nodes now the store is on NFS. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JK7o9EnraydJLhkzaVpLsM --- profiles/candide/config.yaml | 29 +++++++++++++++-------------- tests/helpers/cluster.py | 4 ++-- workflow/README.md | 6 ++++++ 3 files changed, 23 insertions(+), 16 deletions(-) diff --git a/profiles/candide/config.yaml b/profiles/candide/config.yaml index 2cfe8d091..eebe706a3 100644 --- a/profiles/candide/config.yaml +++ b/profiles/candide/config.yaml @@ -9,13 +9,14 @@ default-resources: mem_mb: 2000 runtime: 120 # minutes slurm_account: cusers - # comp nodes have TmpDisk=0, so the tile_shape group (--tmp, see tile.smk's - # TILE_SLURM_EXTRA) lands on pscomp; everything else may use either. slurm_partition: "comp,pscomp" - # n09/n17/n36 follow sp_validation's candide exclusions; n23 has measured - # SLURM job failures (sp_validation commit 2b40c99); n34/n35 must not get - # ShapePipe jobs (n34's /tmp filled with orphaned tile stores). - slurm_extra: "'--exclude=n09,n17,n23,n34,n35,n36'" + # A node hosting a data disk mounts it at /nXXdata and has no /automnt/nXXdata; + # every other node reaches it through the /nXXdata -> /automnt/nXXdata symlink. + # Snakemake resolves the state dir to the /automnt form and puts its Slurm logs + # there, so a job on the node hosting the state dir dies at start (exit 0:53) + # with no log. Campaign state normally sits on /n17data, hence n17. State on + # another disk moves the problem to that disk's node: see workflow/README.md. + slurm_extra: "'--exclude=n17'" software-deployment-method: [apptainer] # tile.smk's /local/scratch (the tile store, ~5.6 GB per real-data tile) is # bound to a shared NFS directory. candide has no /local; node /tmp (~29 GB, @@ -39,16 +40,16 @@ show-failed-logs: true printshellcmds: true rerun-triggers: [mtime, params, code, software-env] # @sc [label:coupling] candide-group-slurm-extra -# tile_shape's rule-level slurm_extra replaces the profile default. Set both -# the --tmp floor and node exclusions identically on all four member rules: -# a group's string resources must agree. Leave rule threads and attempt-scaled -# memory intact. +# tile_shape's rule-level slurm_extra replaces the profile default. On candide +# it carries only the node exclusion: the tile store is on NFS, so tile.smk's +# --tmp floor would just keep the group off the TmpDisk=0 nodes. Set it +# identically on all four member rules: a group's string resources must agree. set-resources: tile_vignets: - slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n34,n35,n36'" + slurm_extra: "'--exclude=n17'" tile_ngmix: - slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n34,n35,n36'" + slurm_extra: "'--exclude=n17'" tile_merge_cats: - slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n34,n35,n36'" + slurm_extra: "'--exclude=n17'" tile_make_cat: - slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n34,n35,n36'" + slurm_extra: "'--exclude=n17'" diff --git a/tests/helpers/cluster.py b/tests/helpers/cluster.py index 1a8addcc5..c770bf6c3 100644 --- a/tests/helpers/cluster.py +++ b/tests/helpers/cluster.py @@ -53,14 +53,14 @@ def srun(args, *, partition="comp", time_limit="00:30:00", cpus=4, extra=None): cpus : int ``--cpus-per-task``. extra : list of str, optional - Extra ``srun`` flags (e.g. ``["--exclude", "n09,n17,n36"]``). + Extra ``srun`` flags (e.g. ``["--nodelist", "n24"]``). """ cmd = [ "srun", f"--partition={partition}", f"--time={time_limit}", f"--cpus-per-task={cpus}", - *(extra or ["--exclude", "n09,n17,n36"]), + *(extra or []), *args, ] return subprocess.run(cmd, check=True, text=True, capture_output=True) diff --git a/workflow/README.md b/workflow/README.md index 294cce234..e5152bd55 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -139,6 +139,12 @@ set `tile_store_root:` in the run config; `sp run` binds it to `/local/scratch` for that campaign. The store names carry a per-campaign hash, so all branches can share one root. +Also on candide, a job cannot start on the node that hosts its state directory +(`SP_STATE_DIR`, default `-state`): Slurm reports exit `0:53` and writes +no log. The profile excludes n17 because state normally sits on `/n17data`. A +campaign with state on, say, `/n23data1` must add `--exclude=n23` to the profile's +`slurm_extra` entries, or move its state directory to another disk. + ## The container image `sp container` owns which image the jobs run inside. Two layers, and the second From d045f80ff8915d0a491d9a75c4de467f00171520 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 4 Nov 2026 09:36:01 -0500 Subject: [PATCH 2/7] candide: no node excludes; tile store on each job's own /scratch The tile store is per-job intermediate data, so bind Slurm's per-job directory on the node's own disk (tile_store_root: /scratch/%u/%j) from the Snakefile, which every job parses in its allocation. This replaces sp's rewrite of the profile and the single user's NFS directory. Slurm logs go to the plain /nXXdata spelling, the one path that kills a job on the node owning the disk, so n17 needs no exclude. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JK7o9EnraydJLhkzaVpLsM --- profiles/candide/config.yaml | 32 ++++++++------------------------ workflow/README.md | 24 +++++++++++++----------- workflow/Snakefile | 11 +++++++++++ workflow/bin/sp | 31 +++++-------------------------- workflow/config.yaml | 10 +++++++--- workflow/rules/tile.smk | 2 +- 6 files changed, 45 insertions(+), 65 deletions(-) diff --git a/profiles/candide/config.yaml b/profiles/candide/config.yaml index eebe706a3..b72681378 100644 --- a/profiles/candide/config.yaml +++ b/profiles/candide/config.yaml @@ -10,24 +10,9 @@ default-resources: runtime: 120 # minutes slurm_account: cusers slurm_partition: "comp,pscomp" - # A node hosting a data disk mounts it at /nXXdata and has no /automnt/nXXdata; - # every other node reaches it through the /nXXdata -> /automnt/nXXdata symlink. - # Snakemake resolves the state dir to the /automnt form and puts its Slurm logs - # there, so a job on the node hosting the state dir dies at start (exit 0:53) - # with no log. Campaign state normally sits on /n17data, hence n17. State on - # another disk moves the problem to that disk's node: see workflow/README.md. - slurm_extra: "'--exclude=n17'" software-deployment-method: [apptainer] -# tile.smk's /local/scratch (the tile store, ~5.6 GB per real-data tile) is -# bound to a shared NFS directory. candide has no /local; node /tmp (~29 GB, -# shared by every job on the node) must not be used; and the writable per-job -# /scratch/$USER/$SLURM_JOB_ID cannot be named here (literal paths only, see -# profiles/nibi/config.yaml; --cleanenv drops SLURM_JOB_ID). The source dir -# must exist. Cost: ngmix reads the store over NFS. A run can move it with -# tile_store_root (workflow/config.yaml). Stores left by killed jobs are only -# swept after a day by a later tile job; check this dir after failed campaigns. # PYTHONPATH names a checkout's src/; workflow/bin/sp rewrites the launch copy. -apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/n17data/mkilbing/astro/repositories/github/shapepipe/src --bind /home,/automnt,/n17data,/n23data1,/n09data --bind /n23data1/mkilbing/shapepipe/tilestore:/local/scratch" +apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/n17data/mkilbing/astro/repositories/github/shapepipe/src --bind /home,/automnt,/n17data,/n23data1,/n09data" latency-wait: 60 # NFS: wait for outputs to appear after a job keep-going: true # a failed job poisons only its cone; siblings run on # Retry transient node failures, including measured process-slot exhaustion @@ -40,16 +25,15 @@ show-failed-logs: true printshellcmds: true rerun-triggers: [mtime, params, code, software-env] # @sc [label:coupling] candide-group-slurm-extra -# tile_shape's rule-level slurm_extra replaces the profile default. On candide -# it carries only the node exclusion: the tile store is on NFS, so tile.smk's -# --tmp floor would just keep the group off the TmpDisk=0 nodes. Set it -# identically on all four member rules: a group's string resources must agree. +# tile.smk's --tmp floor selects nodes by Slurm's configured TmpDisk, which is 0 +# on nodes that do have /scratch. Clear it on all four tile_shape member rules: +# a group's string resources must agree. set-resources: tile_vignets: - slurm_extra: "'--exclude=n17'" + slurm_extra: "''" tile_ngmix: - slurm_extra: "'--exclude=n17'" + slurm_extra: "''" tile_merge_cats: - slurm_extra: "'--exclude=n17'" + slurm_extra: "''" tile_make_cat: - slurm_extra: "'--exclude=n17'" + slurm_extra: "''" diff --git a/workflow/README.md b/workflow/README.md index e5152bd55..97222d031 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -133,17 +133,19 @@ catalogue as a data campaign, written by the same `final_cat_merge` rule: `config/cfis_image_sims/final_cat.param` and the tile count as the file's `n_tiles` attribute (there is no `n_tiles_final.txt`). -On candide, the tile store is not node-local: the profile binds a shared NFS -directory to `/local/scratch`, never the node's `/tmp`. To use another directory, -set `tile_store_root:` in the run config; `sp run` binds it to `/local/scratch` -for that campaign. The store names carry a per-campaign hash, so all branches can -share one root. - -Also on candide, a job cannot start on the node that hosts its state directory -(`SP_STATE_DIR`, default `-state`): Slurm reports exit `0:53` and writes -no log. The profile excludes n17 because state normally sits on `/n17data`. A -campaign with state on, say, `/n23data1` must add `--exclude=n23` to the profile's -`slurm_extra` entries, or move its state directory to another disk. +The tile store holds one tile's vignets and WCS for the length of its `tile_shape` +job, so it wants the fastest disk the job's node has, and `tile_store_root` in the +`machines:` table names it. On candide that's `/scratch/%u/%j`, the directory Slurm +creates on the node's own disk for each job and deletes when the job ends. Never +use `/tmp` on candide: it is ~30 GB shared by everyone on the node. + +On candide, write paths as `/nXXdataN/...`, never `/automnt/nXXdataN/...`: the +`/automnt` form does not exist on the node that owns that disk. `sp` keeps the state +directory and Slurm logs in the plain form, which is what lets jobs run on the node +holding their own campaign. + +Don't exclude candide nodes from the profile without a reproducible failure of a +pinned job on that node. ## The container image diff --git a/workflow/Snakefile b/workflow/Snakefile index bb7c32933..d5bda7bce 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -83,6 +83,17 @@ if _unresolved: f"$variable for machine={MACHINE!r}, " f"input_type={INPUT_TYPE!r}: {', '.join(_unresolved)}. Set them in " f"your run config (SP_RUN_CONFIG).") + +# Every job parses this file on its node, inside its allocation, so a store on +# per-job disk can be bound here. A login shell's XDG_CACHE_HOME may name a +# node-local directory other nodes cannot create. +os.environ.pop("XDG_CACHE_HOME", None) +_tile_store = config.get("tile_store_root") +if _tile_store and ("%j" not in _tile_store or "SLURM_JOB_ID" in os.environ): + _tile_store = (_tile_store.replace("%u", os.environ.get("USER", "")) + .replace("%j", os.environ.get("SLURM_JOB_ID", ""))) + os.makedirs(_tile_store, exist_ok=True) + os.environ["APPTAINER_BIND"] = f"{_tile_store}:/local/scratch" # The external masks are a data-only input: config_tile_Mc.ini's # MASK_EXT_PATHS reads $SP_INPUT_MASKS, and the sims Mc.ini carries no mask # block. Checked here rather than in run_config.REQUIRED because it depends on diff --git a/workflow/bin/sp b/workflow/bin/sp index e75370849..5ff09e9c8 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -130,6 +130,9 @@ fi # --directory moves state, not data: data paths are absolute, and the Snakefile # resolves its own configfile. STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}" +# candide: /automnt/ is absent on the node that owns the disk; / is +# reachable everywhere. +STATE_DIR="${STATE_DIR#/automnt}" [ "$NEEDS_CAMPAIGN" = 0 ] || mkdir -p "$STATE_DIR" # --- the launch code snapshot ---------------------------------------------- @@ -176,31 +179,6 @@ if not n: f.write_text(text) PY - # Move the tile store by rewriting the bind source, not tile_local()'s params: - # params changes invalidate finished tiles (workflow/CONTRACTS). This bind is - # not a rerun trigger. Append it if the profile has no X:/local/scratch bind. - # See workflow/config.yaml for tile_store_root's capacity/latency trade-off. - python - "$SNAPSHOT/profiles/$PROFILE/config.yaml" "$(cfg tile_store_root)" <<'PY' -import os, pathlib, re, sys -f, root = pathlib.Path(sys.argv[1]), sys.argv[2] -if root: - root = str(root) - if not os.path.isabs(root): - sys.exit(f"sp: tile_store_root must be an absolute path, got {root!r}") - os.makedirs(root, exist_ok=True) - bind = f"--bind {root}:/local/scratch" - text, n = re.subn(r'--bind \S+:/local/scratch(?=[\s"])', lambda m: bind, - f.read_text(), count=1) - if not n: - text, n = re.subn(r'(apptainer-args:\s*"[^"]*)"', - lambda m: m.group(1) + " " + bind + '"', text, count=1) - if not n: - sys.exit("sp: tile_store_root is set but the profile has no quoted " - "apptainer-args to bind it with") - f.write_text(text) - print(f"sp: tile store root {root} (bound to /local/scratch)", file=sys.stderr) -PY - python - "$SNAPSHOT/snapshot.json" "$REPO" <<'PY' import json, pathlib, subprocess, sys, time out, repo = pathlib.Path(sys.argv[1]), sys.argv[2] @@ -252,7 +230,8 @@ sm() { SP_PHASE="${SP_PHASE:-passthrough}" \ snakemake --snakefile "$root/workflow/Snakefile" \ --profile "$root/profiles/$PROFILE" \ - --directory "$STATE_DIR" "$@" + --directory "$STATE_DIR" \ + --slurm-logdir "$STATE_DIR/slurm_logs" "$@" } cmd="${1:-}" diff --git a/workflow/config.yaml b/workflow/config.yaml index d8fa2ac45..3f52b637e 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -98,12 +98,14 @@ machines: base_dir: /n17data/UNIONS/WL data: retrieve: vos + tile_store_root: /scratch/%u/%j inputs: masks: $base_dir/masks/dr6-2026-08 catalogues: vos:cfis/tiles_DR6 container: /n17data/cdaley/containers/shapepipe_develop-runtime-20260718.sif image_sims: retrieve: symlink + tile_store_root: /scratch/%u/%j inputs: # $run is the branch dir (e.g. 1p2z_grid_1), so a run config # picks the grid by setting `run:` alone. @@ -225,6 +227,8 @@ ngmix_chunks: 8 # interpolate. # defect_fill: noise -# Tile store (vignets, WCS). Unset: the profile's /local/scratch bind (nibi -# node NVMe; candide a shared NFS directory, see profiles/candide/config.yaml). -# tile_store_root: /n23data1//shapepipe/tilestore +# Tile store (vignets, WCS): intermediates of one tile_shape job, deleted when it +# ends. tile_store_root is the directory bound to /local/scratch in that job, set +# per machine in machines:; %u and %j are the user and Slurm job id. candide uses +# Slurm's per-job directory on each node's own disk. Unset (nibi): the profile's +# /local bind. A literal directory also works, for example to compare disks. diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index b86246557..c5abd8e59 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -92,7 +92,7 @@ def tile_local(tile): Derive a literal path from the tile wildcard and LOCAL_TAG (the run-dir hash), so concurrent campaigns over the same tile do not share a store. See profiles/nibi/config.yaml's /local bind note for container path handling - and bin/sp's tile_store_root handling for the host-side bind. + and the Snakefile's tile_store_root handling for the per-job bind. On nibi the container's /tmp is RAM-backed, so these stores go on the /local/scratch bind instead. From 945780e8c974cdf34e46b8992e3daa760a38fe62 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 4 Nov 2026 09:56:45 -0500 Subject: [PATCH 3/7] candide profile: jobs derive their own source cache Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JK7o9EnraydJLhkzaVpLsM --- profiles/candide/config.yaml | 9 +++++++++ workflow/Snakefile | 7 ++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/profiles/candide/config.yaml b/profiles/candide/config.yaml index b72681378..84c055e08 100644 --- a/profiles/candide/config.yaml +++ b/profiles/candide/config.yaml @@ -11,6 +11,15 @@ default-resources: slurm_account: cusers slurm_partition: "comp,pscomp" software-deployment-method: [apptainer] +# Everything but source-cache, whose path would be the launch's and may be +# node-local; the Snakefile drops XDG_CACHE_HOME so jobs derive their own. +shared-fs-usage: + - persistence + - input-output + - software-deployment + - sources + - storage-local-copies + - software-deployment-cache # PYTHONPATH names a checkout's src/; workflow/bin/sp rewrites the launch copy. apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/n17data/mkilbing/astro/repositories/github/shapepipe/src --bind /home,/automnt,/n17data,/n23data1,/n09data" latency-wait: 60 # NFS: wait for outputs to appear after a job diff --git a/workflow/Snakefile b/workflow/Snakefile index d5bda7bce..56664d542 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -84,10 +84,11 @@ if _unresolved: f"input_type={INPUT_TYPE!r}: {', '.join(_unresolved)}. Set them in " f"your run config (SP_RUN_CONFIG).") -# Every job parses this file on its node, inside its allocation, so a store on -# per-job disk can be bound here. A login shell's XDG_CACHE_HOME may name a -# node-local directory other nodes cannot create. +# A login shell's XDG_CACHE_HOME may name a node-local directory other nodes +# cannot create, and jobs inherit the launch's environment. os.environ.pop("XDG_CACHE_HOME", None) +# Every job parses this file on its node, inside its allocation, so a store on +# per-job disk can be bound here. _tile_store = config.get("tile_store_root") if _tile_store and ("%j" not in _tile_store or "SLURM_JOB_ID" in os.environ): _tile_store = (_tile_store.replace("%u", os.environ.get("USER", "")) From 0e572ffed97fd0e1ec2420bad15f21ba42108681 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 4 Nov 2026 10:14:25 -0500 Subject: [PATCH 4/7] Snakefile: spell the Slurm working directory in the plain form Both the job's working directory and its log derive from workdir_init, so fixing it there replaces the log-directory flag. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JK7o9EnraydJLhkzaVpLsM --- workflow/README.md | 6 +++--- workflow/Snakefile | 5 +++++ workflow/bin/sp | 6 +----- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/workflow/README.md b/workflow/README.md index 97222d031..4168b89a0 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -140,9 +140,9 @@ creates on the node's own disk for each job and deletes when the job ends. Never use `/tmp` on candide: it is ~30 GB shared by everyone on the node. On candide, write paths as `/nXXdataN/...`, never `/automnt/nXXdataN/...`: the -`/automnt` form does not exist on the node that owns that disk. `sp` keeps the state -directory and Slurm logs in the plain form, which is what lets jobs run on the node -holding their own campaign. +`/automnt` form does not exist on the node that owns that disk. The Snakefile spells +Snakemake's working directory in the plain form, so jobs can run on the node holding +their own campaign. Don't exclude candide nodes from the profile without a reproducible failure of a pinned job on that node. diff --git a/workflow/Snakefile b/workflow/Snakefile index 56664d542..f3ad8fa14 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -84,6 +84,11 @@ if _unresolved: f"input_type={INPUT_TYPE!r}: {', '.join(_unresolved)}. Set them in " f"your run config (SP_RUN_CONFIG).") +# candide: Snakemake resolves its working directory to /automnt/, which is +# absent on the node owning the disk, while / is reachable everywhere. The +# Slurm executor uses this value for the job's working directory and its log. +workflow._workdir_init = workflow._workdir_init.removeprefix("/automnt") + # A login shell's XDG_CACHE_HOME may name a node-local directory other nodes # cannot create, and jobs inherit the launch's environment. os.environ.pop("XDG_CACHE_HOME", None) diff --git a/workflow/bin/sp b/workflow/bin/sp index 5ff09e9c8..99b213b36 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -130,9 +130,6 @@ fi # --directory moves state, not data: data paths are absolute, and the Snakefile # resolves its own configfile. STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}" -# candide: /automnt/ is absent on the node that owns the disk; / is -# reachable everywhere. -STATE_DIR="${STATE_DIR#/automnt}" [ "$NEEDS_CAMPAIGN" = 0 ] || mkdir -p "$STATE_DIR" # --- the launch code snapshot ---------------------------------------------- @@ -230,8 +227,7 @@ sm() { SP_PHASE="${SP_PHASE:-passthrough}" \ snakemake --snakefile "$root/workflow/Snakefile" \ --profile "$root/profiles/$PROFILE" \ - --directory "$STATE_DIR" \ - --slurm-logdir "$STATE_DIR/slurm_logs" "$@" + --directory "$STATE_DIR" "$@" } cmd="${1:-}" From a680ae803d10c0c2206e744f18c23e27e952cdd7 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 4 Nov 2026 10:22:01 -0500 Subject: [PATCH 5/7] candide profile: Slurm logs join the plain working directory Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JK7o9EnraydJLhkzaVpLsM --- profiles/candide/config.yaml | 3 +++ workflow/Snakefile | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/profiles/candide/config.yaml b/profiles/candide/config.yaml index 84c055e08..323aa16ef 100644 --- a/profiles/candide/config.yaml +++ b/profiles/candide/config.yaml @@ -2,6 +2,9 @@ # `SP_PROFILE=candide workflow/bin/sp run`. Mirrors profiles/nibi/config.yaml; # only the cluster facts differ, and each is stated where it is set. executor: slurm +# Relative, so it joins the plain-spelled working directory the Snakefile sets +# rather than resolving to /automnt/, absent on the node owning the disk. +slurm-logdir: .snakemake/slurm_logs # candide is a shared ~25-node cluster: cap concurrent submissions well below # what nibi takes. jobs: 150 diff --git a/workflow/Snakefile b/workflow/Snakefile index f3ad8fa14..56f3ecffc 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -86,7 +86,8 @@ if _unresolved: # candide: Snakemake resolves its working directory to /automnt/, which is # absent on the node owning the disk, while / is reachable everywhere. The -# Slurm executor uses this value for the job's working directory and its log. +# Slurm executor uses this value for the job's working directory, and for its log +# through the profile's relative slurm-logdir. workflow._workdir_init = workflow._workdir_init.removeprefix("/automnt") # A login shell's XDG_CACHE_HOME may name a node-local directory other nodes From 49a7a08c51441975898e66c2c5b4f2e2fe7c4161 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 4 Nov 2026 10:32:47 -0500 Subject: [PATCH 6/7] Snakefile: plain working directory for every path handed to jobs Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JK7o9EnraydJLhkzaVpLsM --- workflow/Snakefile | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 56f3ecffc..e7c422e40 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -84,11 +84,13 @@ if _unresolved: f"input_type={INPUT_TYPE!r}: {', '.join(_unresolved)}. Set them in " f"your run config (SP_RUN_CONFIG).") -# candide: Snakemake resolves its working directory to /automnt/, which is -# absent on the node owning the disk, while / is reachable everywhere. The -# Slurm executor uses this value for the job's working directory, and for its log -# through the profile's relative slurm-logdir. +# candide: the working directory resolves to /automnt/, which is absent on +# the node owning the disk, while / is reachable everywhere. The head process +# derives the paths it hands to jobs from it (their working directory, Slurm log +# and wait-for files), so spell it in the plain form before any of them is made. workflow._workdir_init = workflow._workdir_init.removeprefix("/automnt") +_getcwd = os.getcwd +os.getcwd = lambda: _getcwd().removeprefix("/automnt") # A login shell's XDG_CACHE_HOME may name a node-local directory other nodes # cannot create, and jobs inherit the launch's environment. From 960999512ed2fade545265e28b06a891dcab1f0f Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Wed, 4 Nov 2026 12:53:38 -0500 Subject: [PATCH 7/7] Snakefile: scope the working-directory spelling to candide Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01JK7o9EnraydJLhkzaVpLsM --- workflow/Snakefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index e7c422e40..d22fe5e5b 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -88,9 +88,9 @@ if _unresolved: # the node owning the disk, while / is reachable everywhere. The head process # derives the paths it hands to jobs from it (their working directory, Slurm log # and wait-for files), so spell it in the plain form before any of them is made. -workflow._workdir_init = workflow._workdir_init.removeprefix("/automnt") -_getcwd = os.getcwd -os.getcwd = lambda: _getcwd().removeprefix("/automnt") +if MACHINE == "candide": + workflow._workdir_init = workflow._workdir_init.removeprefix("/automnt") + os.getcwd = lambda getcwd=os.getcwd: getcwd().removeprefix("/automnt") # A login shell's XDG_CACHE_HOME may name a node-local directory other nodes # cannot create, and jobs inherit the launch's environment.