diff --git a/profiles/candide/config.yaml b/profiles/candide/config.yaml index bc82c8ee2..2cfe8d091 100644 --- a/profiles/candide/config.yaml +++ b/profiles/candide/config.yaml @@ -13,16 +13,20 @@ default-resources: # TILE_SLURM_EXTRA) lands on pscomp; everything else may use either. slurm_partition: "comp,pscomp" # n09/n17/n36 follow sp_validation's candide exclusions; n23 has measured - # SLURM job failures (sp_validation commit 2b40c99). - slurm_extra: "'--exclude=n09,n17,n23,n36'" + # SLURM job failures (sp_validation commit 2b40c99); n34/n35 must not get + # ShapePipe jobs (n34's /tmp filled with orphaned tile stores). + slurm_extra: "'--exclude=n09,n17,n23,n34,n35,n36'" software-deployment-method: [apptainer] -# Bind node /tmp (~29 GB free, local disk) to tile.smk's /local/scratch: -# candide has no /local, and its writable /scratch path requires a job ID. -# Variable expansion limits are documented in profiles/nibi/config.yaml. -# The disk holds ~5 real-data tile stores (~5.6 GB each); dense image-sim runs -# can set tile_store_root in the run config (workflow/config.yaml). +# tile.smk's /local/scratch (the tile store, ~5.6 GB per real-data tile) is +# bound to a shared NFS directory. candide has no /local; node /tmp (~29 GB, +# shared by every job on the node) must not be used; and the writable per-job +# /scratch/$USER/$SLURM_JOB_ID cannot be named here (literal paths only, see +# profiles/nibi/config.yaml; --cleanenv drops SLURM_JOB_ID). The source dir +# must exist. Cost: ngmix reads the store over NFS. A run can move it with +# tile_store_root (workflow/config.yaml). Stores left by killed jobs are only +# swept after a day by a later tile job; check this dir after failed campaigns. # PYTHONPATH names a checkout's src/; workflow/bin/sp rewrites the launch copy. -apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/n17data/mkilbing/astro/repositories/github/shapepipe/src --bind /home,/automnt,/n17data,/n23data1,/n09data --bind /tmp:/local/scratch" +apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/n17data/mkilbing/astro/repositories/github/shapepipe/src --bind /home,/automnt,/n17data,/n23data1,/n09data --bind /n23data1/mkilbing/shapepipe/tilestore:/local/scratch" latency-wait: 60 # NFS: wait for outputs to appear after a job keep-going: true # a failed job poisons only its cone; siblings run on # Retry transient node failures, including measured process-slot exhaustion @@ -41,10 +45,10 @@ rerun-triggers: [mtime, params, code, software-env] # memory intact. set-resources: tile_vignets: - slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n36'" + slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n34,n35,n36'" tile_ngmix: - slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n36'" + slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n34,n35,n36'" tile_merge_cats: - slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n36'" + slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n34,n35,n36'" tile_make_cat: - slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n36'" + slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n34,n35,n36'" diff --git a/workflow/README.md b/workflow/README.md index 897e2a204..294cce234 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -133,11 +133,11 @@ catalogue as a data campaign, written by the same `final_cat_merge` rule: `config/cfis_image_sims/final_cat.param` and the tile count as the file's `n_tiles` attribute (there is no `n_tiles_final.txt`). -On candide, the node-local tile store (bound from the node's 31 GB `/tmp`) does not -hold several dense image-sim tiles at once. Set `tile_store_root:` in the run -config to a shared directory; `sp run` binds it to `/local/scratch` for that -campaign. The store names carry a per-campaign hash, so all branches can share one -root. +On candide, the tile store is not node-local: the profile binds a shared NFS +directory to `/local/scratch`, never the node's `/tmp`. To use another directory, +set `tile_store_root:` in the run config; `sp run` binds it to `/local/scratch` +for that campaign. The store names carry a per-campaign hash, so all branches can +share one root. ## The container image diff --git a/workflow/config.yaml b/workflow/config.yaml index d2f0c37bb..d8fa2ac45 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -225,7 +225,6 @@ ngmix_chunks: 8 # interpolate. # defect_fill: noise -# Node-local tile store (vignets, WCS). Unset: the profile's /local/scratch -# bind (nibi NVMe, candide node /tmp). Dense image-sim campaigns on candide -# want a shared directory instead. +# Tile store (vignets, WCS). Unset: the profile's /local/scratch bind (nibi +# node NVMe; candide a shared NFS directory, see profiles/candide/config.yaml). # tile_store_root: /n23data1//shapepipe/tilestore