diff --git a/BENCHMARKING.md b/BENCHMARKING.md index 23843c6..14ac287 100644 --- a/BENCHMARKING.md +++ b/BENCHMARKING.md @@ -243,7 +243,7 @@ config/loss/ckpt recorded in the JSONs), and a no-op The mirror of A's ablation on the MONAI backbone: C trained with every hyperparameter at C's own default (its original study protocol from -`scripts_monai/02_monai_train.py`, nothing nnU-derived) — BatchNorm +`scripts_monai/07_train.py`, nothing nnU-derived) — BatchNorm `UNet(16→128)` with MONAI default init, LeakyReLU (C's own default act), per-sample 0.5CE+0.5mean-dice loss (WeightedDiceCELoss, no deep supervision), AdamW 3e-4 (wd 1e-4) + warmup/cosine, grad clip 5.0, @@ -314,7 +314,7 @@ Artifacts: `results/benchmark_full/benchmark_full_C_legacy_std_100ep.json` 0.4002 @100ep record; `--legacy` reverts). Checkpoint conventions: see the A and C ablation sections. - Study entrypoints match the record configs (2026-10-07): `scripts/07_train.py` - (A) was already A-native all-defaults; `scripts_monai/02_monai_train.py` + (A) was already A-native all-defaults; `scripts_monai/07_train.py` (C) now trains B-aligned by default (its record config) — `--legacy` reverts to the pre-2026-10-07 config. C checkpoints carry a `b_aligned` marker that 03/04 read for the Z-score input; in C's B-aligned study mode pseudo rows diff --git a/README.md b/README.md index 67e5565..bdeb95b 100644 --- a/README.md +++ b/README.md @@ -95,6 +95,10 @@ Run the full study: python scripts/09_run_iterative.py --rounds 4 --gpus 3 ``` +Per-round test results are saved to `results/round{k}_test.json`, and +`results/iterative_table.jsonl` + `results/iterative_dice.png` are rewritten +after every round's evaluation. + Individual stages are standalone, e.g.: ```bash @@ -140,12 +144,12 @@ Dataset: `Dataset210_NTUH_T1C_PL`, single channel `t1c`, 2 classes | # | Script | Purpose | |---|---|---| -| 01 | `01_nnu_prepare_dataset.py` | Build/rebuild the raw dataset (symlinked **native** `imagesTr`, `labelsTr` 0/1 + `dataset.json`) from a rows jsonl | -| 02 | `02_nnu_plan_preprocess.py` | Plan + preprocess (`--clean`), then write subject-level `splits_final.json` | -| 03 | `03_nnu_train.py` | Train one round via `NTUHLPLTrainer` (DDP `-num_gpus`), optional warm start | -| 04 | `04_nnu_pseudo_label.py` | Multi-GPU `nnUNetv2_predict` on native pool volumes + selection gates (physical mm) | -| 05 | `05_nnu_eval_test.py` | Held-out test evaluation on native volumes (probability Dice@0.5 + hard-seg Dice) | -| 06 | `06_nnu_run_iterative.py` | Orchestrates rounds 0…K, table + plot | +| 01 | `01_prepare_dataset.py` | Build/rebuild the raw dataset (symlinked **native** `imagesTr`, `labelsTr` 0/1 + `dataset.json`) from a rows jsonl | +| 02 | `02_plan_preprocess.py` | Plan + preprocess (`--clean`), then write subject-level `splits_final.json` | +| 06 | `06_pseudo_label.py` | Multi-GPU `nnUNetv2_predict` on native pool volumes + selection gates (physical mm) | +| 07 | `07_train.py` | Train one round via `NTUHLPLTrainer` (DDP `-num_gpus`), optional warm start | +| 08 | `08_eval.py` | Held-out test evaluation on native volumes (probability Dice@0.5 + hard-seg Dice) | +| 09 | `09_run_iterative.py` | Orchestrates rounds 0…K, table + plot | | — | `trainers/ntuh_pl_trainer.py` | `NTUHLPLTrainer`: env-driven epochs/LR + full-weight warm start | | — | `nnu_common.py` | Shared paths, env, dataset/split helpers, native-grid selection + consistency | @@ -210,25 +214,26 @@ across visits. Full study from the repo root: ```bash -python scripts_nnu/06_nnu_run_iterative.py --rounds 4 --gpus 3 +python scripts_nnu/09_run_iterative.py --rounds 4 --gpus 3 ``` Defaults: baseline 250 epochs @ 1e-2; warm-started rounds 75 epochs @ 1e-3. Pseudo-label gates match Pipeline A (`--tau-pos 0.95`, `--min-cc-frac 0.2`, `--neg-frac 0.9`, volume p2–p98). Optional flags: `--no-tta`, `--no-neg-pseudo`, and the gate overrides. Outputs to `results/round{k}_test_nnu.json`, -`results/iterative_table_nnu.jsonl`, `results/iterative_dice_nnu.png`; per-round +`results/iterative_table_nnu.jsonl` (rewritten after every round's evaluation), +`results/iterative_dice_nnu.png`; per-round checkpoints snapshotted to `runs_nnu/round{k}/best_nnu.pth`. Individual stages: ```bash -python scripts_nnu/01_nnu_prepare_dataset.py --rows -python scripts_nnu/02_nnu_plan_preprocess.py --val data/manifests/split_val.jsonl -python scripts_nnu/03_nnu_train.py --gpus 3 --epochs 250 --lr 1e-2 -python scripts_nnu/04_nnu_pseudo_label.py --pool data/manifests/unlabeled_pool.jsonl \ +python scripts_nnu/01_prepare_dataset.py --rows +python scripts_nnu/02_plan_preprocess.py --val data/manifests/split_val.jsonl +python scripts_nnu/07_train.py --gpus 3 --epochs 250 --lr 1e-2 +python scripts_nnu/06_pseudo_label.py --pool data/manifests/unlabeled_pool.jsonl \ --out data/pseudo_nnu/round1 --gpus 3 -python scripts_nnu/05_nnu_eval_test.py --rows data/manifests/split_test.jsonl \ +python scripts_nnu/08_eval.py --rows data/manifests/split_test.jsonl \ --out results/round0_test_nnu.json --gpus 3 ``` @@ -278,11 +283,11 @@ directly comparable to Pipelines A and B. | # | Script | Purpose | |---|---|---| -| 01 | `01_monai_build_rows.py` | Assemble a round's training manifest (labeled + accepted pseudo rows, weighted, de-duped by key) | -| 02 | `02_monai_train.py` | DDP training of the MONAI UNet for one round (torchrun), full-weight warm start (same config mode), auto-resume | -| 03 | `03_monai_pseudo_label.py` | Sliding-window + TTA prediction of the remaining pool, selection gates + consistency filter | -| 04 | `04_monai_eval_test.py` | Held-out test evaluation (Dice from probability map @0.5) | -| 05 | `05_monai_run_iterative.py` | Orchestrates rounds 0…K, table + plot | +| 01 | `01_build_rows.py` | Assemble a round's training manifest (labeled + accepted pseudo rows, weighted, de-duped by key) | +| 06 | `06_pseudo_label.py` | Sliding-window + TTA prediction of the remaining pool, selection gates + consistency filter | +| 07 | `07_train.py` | DDP training of the MONAI UNet for one round (torchrun), full-weight warm start (same config mode), auto-resume | +| 08 | `08_eval.py` | Held-out test evaluation (Dice from probability map @0.5) | +| 09 | `09_run_iterative.py` | Orchestrates rounds 0…K, table + plot | | — | `monai_common.py` | Network (legacy + B-aligned) / losses / transform chain / inference + distributed helpers | ### Running @@ -290,7 +295,7 @@ directly comparable to Pipelines A and B. Full study from the repo root: ```bash -python scripts_monai/05_monai_run_iterative.py --rounds 4 --gpus 3 +python scripts_monai/09_run_iterative.py --rounds 4 --gpus 3 ``` Defaults: B-aligned protocol (SGD 1e-2 + PolyLR, restarted per round); baseline @@ -300,24 +305,25 @@ pre-2026-10-07 config (baseline 100 epochs @ 3e-4, warm-started rounds Pipelines A/B (`--tau-pos 0.95`, `--min-cc-frac 0.2`, `--neg-frac 0.9`, volume p2–p98). Optional flags: `--no-tta`, `--no-neg-pseudo`, `--legacy`, and the gate overrides. Outputs to -`results/round{k}_test_monai.json`, `results/iterative_table_monai.jsonl`, +`results/round{k}_test_monai.json`, `results/iterative_table_monai.jsonl` +(rewritten after every round's evaluation), `results/iterative_dice_monai.png`; per-round checkpoints in `runs_monai/round{k}/best.pt`. Individual stages: ```bash -python scripts_monai/01_monai_build_rows.py --train data/manifests/split_train.jsonl \ +python scripts_monai/01_build_rows.py --train data/manifests/split_train.jsonl \ --accepted data/pseudo_monai/round1/accepted.jsonl \ --out data/pseudo_monai/round1_rows.jsonl -torchrun --standalone --nproc_per_node 3 scripts_monai/02_monai_train.py \ +torchrun --standalone --nproc_per_node 3 scripts_monai/07_train.py \ --rows data/pseudo_monai/round1_rows.jsonl --val data/manifests/split_val.jsonl \ --epochs 30 --batch 3 --ckpt-dir runs_monai/round1 \ --pretrained runs_monai/round0/best.pt -torchrun --standalone --nproc_per_node 3 scripts_monai/03_monai_pseudo_label.py \ +torchrun --standalone --nproc_per_node 3 scripts_monai/06_pseudo_label.py \ --ckpt runs_monai/round1/best.pt --pool data/manifests/unlabeled_pool.jsonl \ --out data/pseudo_monai/round2 --already data/pseudo_monai/added_keys.jsonl -torchrun --standalone --nproc_per_node 3 scripts_monai/04_monai_eval_test.py \ +torchrun --standalone --nproc_per_node 3 scripts_monai/08_eval.py \ --rows data/manifests/split_test.jsonl --ckpt runs_monai/round1/best.pt \ --out results/round1_test_monai.json ``` \ No newline at end of file diff --git a/scripts/09_run_iterative.py b/scripts/09_run_iterative.py index 156e6de..5b08697 100644 --- a/scripts/09_run_iterative.py +++ b/scripts/09_run_iterative.py @@ -39,6 +39,54 @@ def torchrun(nproc, script, extra): return f"{tr} --standalone --nproc_per_node {nproc} {script} {extra}" +def build_table(args): + """Scan per-round test results, rewrite results/iterative_table.jsonl + plot. + + Called after every round's evaluation so each round's performance is saved + incrementally (survives interrupted runs). + """ + results_dir = f"{ROOT}/results" + table = [] + for k in range(args.rounds + 1): + f = os.path.join(results_dir, f"round{k}_test.json") + if not os.path.exists(f): + continue + r = json.load(open(f)) + row = {"round": k, "test_dice": round(r["dice"], 4), "n_test": r["n"], + "ckpt_epoch": r.get("epoch")} + pf = os.path.join(DATA, "pseudo", f"round{k}", "summary.json") if k > 0 else None + if pf and os.path.exists(pf): + s = json.load(open(pf)) + row.update({"n_pos": s["n_pos"], "n_neg": s["n_neg"], + "n_rej_cons": s["n_rejected_consistency"], + "pos_vol_med_mm3": s["pos_vol_mm3"]["med"]}) + sf = os.path.join(ROOT, "runs", f"round{k}", "final.pt") + row["ckpt"] = sf if os.path.exists(sf) else "" + table.append(row) + save_jsonl(table, os.path.join(results_dir, "iterative_table.jsonl")) + if not table: + return table + print(json.dumps(table, indent=1)) + try: + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + rs = [t["round"] for t in table] + ds = [t["test_dice"] for t in table] + plt.figure(figsize=(6, 4)) + plt.plot(rs, ds, "o-") + plt.xlabel("pseudo-labeling round") + plt.ylabel("holdout tumor Dice") + for x, y in zip(rs, ds): + plt.annotate(f"{y:.3f}", (x, y), textcoords="offset points", xytext=(0, 8), fontsize=8) + plt.grid(alpha=0.3) + plt.tight_layout() + plt.savefig(os.path.join(results_dir, "iterative_dice.png"), dpi=150) + except Exception as e: # noqa + print("plot failed:", repr(e)) + return table + + def main(): ap = argparse.ArgumentParser() ap.add_argument("--rounds", type=int, default=4) @@ -67,6 +115,7 @@ def main(): f"--rows {train_f} --val {val_f} --epochs {args.base_epochs} --lr {args.lr} " f"--batch {args.batch} --ckpt-dir runs/round0"), f"{main_log}") run(f"python scripts/08_eval.py --rows {test_f} --ckpt runs/round0/best.pt", f"{main_log}") + build_table(args) for k in range(1, args.rounds + 1): pdir = os.path.join(DATA, "pseudo/round" + str(k)) @@ -91,42 +140,10 @@ def main(): f"--lr {args.round_lr} --batch {args.batch} --resume runs/round{k-1}/best.pt " f"--ckpt-dir runs/round{k} --val-every 1"), f"{main_log}") run(f"python scripts/08_eval.py --rows {test_f} --ckpt runs/round{k}/best.pt", f"{main_log}") + build_table(args) # ---- report ---- - table = [] - for k in list(range(args.rounds + 1)): - f = os.path.join(results_dir, f"round{k}_test.json") - if os.path.exists(f): - r = json.load(open(f)) - table.append({"round": k, "test_dice": round(r["dice"], 4), "n_test": r["n"], - "ckpt_epoch": r.get("epoch")}) - pf = os.path.join(DATA, f"pseudo/round{k}/summary.json") if k > 0 else None - if pf and os.path.exists(pf): - s = json.load(open(pf)) - table[-1].update({"n_pos": s["n_pos"], "n_neg": s["n_neg"], - "n_rej_cons": s["n_rejected_consistency"], - "pos_vol_med_mm3": s["pos_vol_mm3"]["med"]}) - sf = os.path.join(ROOT, f"runs/round{k}/final.pt") - table[-1]["ckpt"] = sf if os.path.exists(sf) else "" - save_jsonl(table, os.path.join(results_dir, "iterative_table.jsonl")) - print(json.dumps(table, indent=1)) - try: - import matplotlib - matplotlib.use("Agg") - import matplotlib.pyplot as plt - rs = [t["round"] for t in table] - ds = [t["test_dice"] for t in table] - plt.figure(figsize=(6, 4)) - plt.plot(rs, ds, "o-") - plt.xlabel("pseudo-labeling round") - plt.ylabel("holdout tumor Dice") - for x, y in zip(rs, ds): - plt.annotate(f"{y:.3f}", (x, y), textcoords="offset points", xytext=(0, 8), fontsize=8) - plt.grid(alpha=0.3) - plt.tight_layout() - plt.savefig(os.path.join(results_dir, "iterative_dice.png"), dpi=150) - except Exception as e: # noqa - print("plot failed:", repr(e)) + build_table(args) if __name__ == "__main__": diff --git a/scripts_monai/01_build_rows.py b/scripts_monai/01_build_rows.py index 5a7378c..8404ad3 100644 --- a/scripts_monai/01_build_rows.py +++ b/scripts_monai/01_build_rows.py @@ -3,12 +3,12 @@ base = labeled patient-level train split (w=1.0) — the same train/val usage as Pipeline A: the val split is never trained on (it is the internal-validation set). Accepted pseudo-label rows (pos + neg) are appended with w= -(negative rows point at the zero mask written by stage 03), de-duplicated by +(negative rows point at the zero mask written by stage 06), de-duplicated by key (first occurrence wins). Every row must point at existing pimg/plabel niftis. Usage: - python scripts_monai/01_monai_build_rows.py \ + python scripts_monai/01_build_rows.py \ --train data/manifests/split_train.jsonl \ --accepted data/pseudo_monai/round1/accepted.jsonl \ --pseudo-weight 0.3 --out data/pseudo_monai/round1_rows.jsonl diff --git a/scripts_monai/06_pseudo_label.py b/scripts_monai/06_pseudo_label.py index 3f0bee0..342bc28 100644 --- a/scripts_monai/06_pseudo_label.py +++ b/scripts_monai/06_pseudo_label.py @@ -1,6 +1,6 @@ """Pseudo-label the unlabeled pool with the MONAI round model (torchrun; rank = GPU). -torchrun --standalone --nproc_per_node 3 scripts_monai/03_monai_pseudo_label.py \ +torchrun --standalone --nproc_per_node 3 scripts_monai/06_pseudo_label.py \ --ckpt runs_monai/round0/best.pt --pool data/manifests/unlabeled_pool.jsonl \ --out data/pseudo_monai/round1 [--already data/pseudo_monai/added_keys.jsonl] @@ -18,8 +18,8 @@ mask for neg), /accepted.jsonl, /summary.json. Per-rank shards are flushed to /part{rank}.jsonl so interrupted runs resume. Usage: - python scripts_monai/03_monai_pseudo_label.py - torchrun --standalone --nproc_per_node 3 scripts_monai/03_monai_pseudo_label.py ... + python scripts_monai/06_pseudo_label.py + torchrun --standalone --nproc_per_node 3 scripts_monai/06_pseudo_label.py ... """ import argparse import os diff --git a/scripts_monai/07_train.py b/scripts_monai/07_train.py index 222bd7c..1ff7825 100644 --- a/scripts_monai/07_train.py +++ b/scripts_monai/07_train.py @@ -1,6 +1,6 @@ """MONAI DDP training entrypoint for one pseudo-labeling round (torchrun). -torchrun --standalone --nproc_per_node 3 scripts_monai/02_monai_train.py \ +torchrun --standalone --nproc_per_node 3 scripts_monai/07_train.py \ --rows data/pseudo_monai/round0_rows.jsonl --val data/manifests/split_val.jsonl \ --epochs 100 --batch 3 --ckpt-dir runs_monai/round0 \ [--pretrained runs_monai/round0/best.pt] diff --git a/scripts_monai/08_eval.py b/scripts_monai/08_eval.py index 874a9e5..2dbab4e 100644 --- a/scripts_monai/08_eval.py +++ b/scripts_monai/08_eval.py @@ -1,6 +1,6 @@ """Held-out test evaluation of a MONAI round model (torchrun; rank = GPU). -torchrun --standalone --nproc_per_node 3 scripts_monai/04_monai_eval_test.py \ +torchrun --standalone --nproc_per_node 3 scripts_monai/08_eval.py \ --rows data/manifests/split_test.jsonl --ckpt runs_monai/round0/best.pt \ --out results/round0_test_monai.json [--no-tta] @@ -12,8 +12,8 @@ Per-rank results are appended to _part{rank}.jsonl for resume; rank 0 merges them into (json) + _per_row.jsonl. Usage: - python scripts_monai/04_monai_eval_test.py - torchrun --standalone --nproc_per_node 3 scripts_monai/04_monai_eval_test.py ... + python scripts_monai/08_eval.py + torchrun --standalone --nproc_per_node 3 scripts_monai/08_eval.py ... """ import argparse import os diff --git a/scripts_monai/09_run_iterative.py b/scripts_monai/09_run_iterative.py index f8171e5..d94bb12 100644 --- a/scripts_monai/09_run_iterative.py +++ b/scripts_monai/09_run_iterative.py @@ -1,5 +1,5 @@ """Orchestrates the MONAI iterative pseudo-labeling study (parallel to -scripts/09 and scripts_nnu/06). +scripts/09 and scripts_nnu/09). Round 0: train the MONAI UNet on the labeled patient-level train split (internal validation = held-out patient-level val split, subject-disjoint). @@ -15,7 +15,7 @@ Config: rounds train with the B-aligned protocol (C's record benchmark config, --legacy (pre-2026-10-07 config: AdamW at --base-lr/--round-lr + per-sample weighted Dice+CE with pseudo-row down-weighting). -Usage: python scripts_monai/05_monai_run_iterative.py [--rounds 4] [--gpus 3] [--legacy] +Usage: python scripts_monai/09_run_iterative.py [--rounds 4] [--gpus 3] [--legacy] """ import os import sys @@ -101,7 +101,7 @@ def main(): extra += " --no-neg" for f in accepted: extra += f" --accepted {f}" - py("01_monai_build_rows.py", extra) + py("01_build_rows.py", extra) def train(round, rows_f, epochs, lr, warmstart, val_every): extra = (f"--rows {rows_f} --val {val_f} --epochs {epochs} --lr {lr} " @@ -111,7 +111,7 @@ def main(): extra += f" --pretrained {warmstart}" if args.legacy: extra += " --legacy" - trrun("02_monai_train.py", extra, os.path.join(logdir, f"train_r{round}.log")) + trrun("07_train.py", extra, os.path.join(logdir, f"train_r{round}.log")) def predict_pool(round, ckpt, added=None): out = os.path.join(pdir_root, f"round{round}") @@ -123,7 +123,7 @@ def main(): extra += f" --already {added}" if args.no_tta: extra += " --no-tta" - trrun("03_monai_pseudo_label.py", extra, os.path.join(logdir, f"pseudo_r{round}.log")) + trrun("06_pseudo_label.py", extra, os.path.join(logdir, f"pseudo_r{round}.log")) return os.path.join(out, "accepted.jsonl") def eval_test(round, ckpt): @@ -131,12 +131,52 @@ def main(): extra = f"--rows {test_f} --ckpt {ckpt} --out {out}" if args.no_tta: extra += " --no-tta" - trrun("04_monai_eval_test.py", extra, os.path.join(logdir, f"eval_r{round}.log")) + trrun("08_eval.py", extra, os.path.join(logdir, f"eval_r{round}.log")) return out def best(round): return os.path.join(ROOT, "runs_monai", f"round{round}", "best.pt") + def build_table(): + # Rewrite the per-round table + plot; called after every round's eval so + # each round's performance is saved incrementally (survives interruptions). + table = [] + for k in range(args.rounds + 1): + resf = os.path.join(results_dir, f"round{k}_test_monai.json") + if not os.path.exists(resf): + continue + r = json.load(open(resf)) + row = {"round": k, "test_dice": round(r["dice"], 4), "n_test": r["n"], "ckpt": best(k)} + pf = os.path.join(pdir_root, f"round{k}", "summary.json") + if k > 0 and os.path.exists(pf): + s = json.load(open(pf)) + row.update({"n_pos": s["n_pos"], "n_neg": s["n_neg"], + "n_rej_cons": s["n_rejected_consistency"], "n_error": s["n_error"], + "pos_vol_med_mm3": s["pos_vol_mm3"]["med"]}) + table.append(row) + save_jsonl(table, os.path.join(results_dir, "iterative_table_monai.jsonl")) + if not table: + return table + print(json.dumps(table, indent=1)) + try: + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + rs = [t["round"] for t in table] + ds = [t["test_dice"] for t in table] + plt.figure(figsize=(6, 4)) + plt.plot(rs, ds, "o-") + plt.xlabel("pseudo-labeling round") + plt.ylabel("holdout tumor Dice (MONAI)") + for x, y in zip(rs, ds): + plt.annotate(f"{y:.3f}", (x, y), textcoords="offset points", xytext=(0, 8), fontsize=8) + plt.grid(alpha=0.3) + plt.tight_layout() + plt.savefig(os.path.join(results_dir, "iterative_dice_monai.png"), dpi=150) + except Exception as e: # noqa + print("plot failed:", repr(e)) + return table + # ---- round 0: baseline ---- rows0 = os.path.join(pdir_root, "round0_rows.jsonl") build_rows(rows0, []) @@ -144,6 +184,7 @@ def main(): accepted_f = [predict_pool(1, best(0))] save_jsonl(load_jsonl(accepted_f[0]), added_f) eval_test(0, best(0)) + build_table() for k in range(1, args.rounds + 1): rows_f = os.path.join(pdir_root, f"round{k}_rows.jsonl") @@ -152,41 +193,10 @@ def main(): accepted_f.append(predict_pool(k + 1, best(k), added_f)) save_jsonl([r for f in accepted_f for r in load_jsonl(f)], added_f) eval_test(k, best(k)) + build_table() - # ---- report (same layout as scripts/09 and scripts_nnu/06) ---- - table = [] - for k in range(args.rounds + 1): - resf = os.path.join(results_dir, f"round{k}_test_monai.json") - if not os.path.exists(resf): - continue - r = json.load(open(resf)) - row = {"round": k, "test_dice": round(r["dice"], 4), "n_test": r["n"], "ckpt": best(k)} - pf = os.path.join(pdir_root, f"round{k}", "summary.json") - if k > 0 and os.path.exists(pf): - s = json.load(open(pf)) - row.update({"n_pos": s["n_pos"], "n_neg": s["n_neg"], - "n_rej_cons": s["n_rejected_consistency"], "n_error": s["n_error"], - "pos_vol_med_mm3": s["pos_vol_mm3"]["med"]}) - table.append(row) - save_jsonl(table, os.path.join(results_dir, "iterative_table_monai.jsonl")) - print(json.dumps(table, indent=1)) - try: - import matplotlib - matplotlib.use("Agg") - import matplotlib.pyplot as plt - rs = [t["round"] for t in table] - ds = [t["test_dice"] for t in table] - plt.figure(figsize=(6, 4)) - plt.plot(rs, ds, "o-") - plt.xlabel("pseudo-labeling round") - plt.ylabel("holdout tumor Dice (MONAI)") - for x, y in zip(rs, ds): - plt.annotate(f"{y:.3f}", (x, y), textcoords="offset points", xytext=(0, 8), fontsize=8) - plt.grid(alpha=0.3) - plt.tight_layout() - plt.savefig(os.path.join(results_dir, "iterative_dice_monai.png"), dpi=150) - except Exception as e: # noqa - print("plot failed:", repr(e)) + # ---- report (same layout as scripts/09 and scripts_nnu/09) ---- + build_table() if __name__ == "__main__": diff --git a/scripts_nnu/01_prepare_dataset.py b/scripts_nnu/01_prepare_dataset.py index 012548d..3aa1149 100644 --- a/scripts_nnu/01_prepare_dataset.py +++ b/scripts_nnu/01_prepare_dataset.py @@ -7,7 +7,7 @@ preprocessing; imagesTr entries are symlinks to the native volumes, labelsTr entries are symlinks when already 0/1 on the image grid, else nearest-warped + binarized copies (label fix-up only). -Usage: python scripts_nnu/01_nnu_prepare_dataset.py --rows +Usage: python scripts_nnu/01_prepare_dataset.py --rows """ import argparse import os diff --git a/scripts_nnu/02_plan_preprocess.py b/scripts_nnu/02_plan_preprocess.py index 17b86de..814444e 100644 --- a/scripts_nnu/02_plan_preprocess.py +++ b/scripts_nnu/02_plan_preprocess.py @@ -2,7 +2,7 @@ subject-level splits_final.json: val = split_val cases, train = the rest (all base + pseudo cases), same 5-fold content so -f 0 is the one we use. -Usage: python scripts_nnu/02_nnu_plan_preprocess.py --val data/manifests/split_val.jsonl [--npp 8] +Usage: python scripts_nnu/02_plan_preprocess.py --val data/manifests/split_val.jsonl [--npp 8] """ import argparse import os @@ -23,7 +23,7 @@ def main(): run(plan_preprocess_cmd(args.npp), log, env=nnu_env()) val_keys = [r["key"] for r in load_jsonl(args.val)] all_keys = dataset_case_keys() - assert all_keys, "raw dataset is empty; run 01_nnu_prepare_dataset.py first" + assert all_keys, "raw dataset is empty; run 01_prepare_dataset.py first" assert set(val_keys) <= set(all_keys), f"val keys not in dataset: {set(val_keys) - set(all_keys)}" n_tr, n_va = write_splits(all_keys, val_keys) print(f"[nnu:plan] splits_final.json written to {preproc_ds()}: train={n_tr} val={n_va}", flush=True) diff --git a/scripts_nnu/06_pseudo_label.py b/scripts_nnu/06_pseudo_label.py index 8319286..8bc8e58 100644 --- a/scripts_nnu/06_pseudo_label.py +++ b/scripts_nnu/06_pseudo_label.py @@ -17,7 +17,7 @@ Writes /rows.jsonl, /_label.nii.gz (native grid), /accepted. /summary.json. Usage: - python scripts_nnu/04_nnu_pseudo_label.py --pool data/manifests/unlabeled_pool.jsonl \ + python scripts_nnu/06_pseudo_label.py --pool data/manifests/unlabeled_pool.jsonl \ --out data/pseudo_nnu/round1 --gpus 3 """ import argparse diff --git a/scripts_nnu/07_train.py b/scripts_nnu/07_train.py index b7d68f1..a16cd6b 100644 --- a/scripts_nnu/07_train.py +++ b/scripts_nnu/07_train.py @@ -4,7 +4,7 @@ Epochs / initial lr / warm-start checkpoint are passed to NTUHLPLTrainer via NNU_PL_* env vars (see scripts_nnu/trainers/ntuh_pl_trainer.py). Usage: - nnUNetv2_train wrapper: python scripts_nnu/03_nnu_train.py \ + nnUNetv2_train wrapper: python scripts_nnu/07_train.py \ --gpus 3 --epochs 250 --lr 1e-2 [--warmstart runs_nnu/round0/best_nnu.pth] """ import argparse diff --git a/scripts_nnu/08_eval.py b/scripts_nnu/08_eval.py index a6cf0d2..32875c7 100644 --- a/scripts_nnu/08_eval.py +++ b/scripts_nnu/08_eval.py @@ -10,7 +10,7 @@ grid, so we score directly there: Ground truth is the row's native label. Writes: json + .json -> per_row jsonl alongside. -Usage: python scripts_nnu/05_nnu_eval_test.py --rows data/manifests/split_test.jsonl \ +Usage: python scripts_nnu/08_eval.py --rows data/manifests/split_test.jsonl \ --out results/round0_test_nnu.json --gpus 3 """ import argparse diff --git a/scripts_nnu/09_run_iterative.py b/scripts_nnu/09_run_iterative.py index 91f3e31..6233f21 100644 --- a/scripts_nnu/09_run_iterative.py +++ b/scripts_nnu/09_run_iterative.py @@ -8,7 +8,7 @@ Round k: dataset grows with accepted pseudo-labels (pos + neg) from rounds 1..k; unlabeled pool (gates identical to scripts/06) and evaluate the model on the held-out patient-level test split. -Usage: python scripts_nnu/06_nnu_run_iterative.py [--rounds 4] [--gpus 3] +Usage: python scripts_nnu/09_run_iterative.py [--rounds 4] [--gpus 3] """ import os import sys @@ -71,16 +71,16 @@ def main(): def prepare(rows): f = os.path.join(pdir_root, "current_rows.jsonl") save_jsonl(rows, f) - py("01_nnu_prepare_dataset.py", f"--rows {f}") + py("01_prepare_dataset.py", f"--rows {f}") def plan(): - py("02_nnu_plan_preprocess.py", f"--val {val_f}") + py("02_plan_preprocess.py", f"--val {val_f}") def train(round, epochs, lr, warmstart): extra = f"--gpus {args.gpus} --epochs {epochs} --lr {lr} --log {os.path.join(logdir, f'nnu_train_r{round}.log')}" if warmstart: extra += f" --warmstart {warmstart}" - py("03_nnu_train.py", extra) + py("07_train.py", extra) dst = os.path.join(ROOT, "runs_nnu", f"round{round}", "best_nnu.pth") os.makedirs(os.path.dirname(dst), exist_ok=True) shutil.copy2(best_ckpt(), dst) @@ -95,16 +95,57 @@ def main(): extra += " --no-tta" if added: extra += f" --already {added_f}" - py("04_nnu_pseudo_label.py", extra) + py("06_pseudo_label.py", extra) return os.path.join(pdir_root, f"round{round}", "accepted.jsonl") def eval_test(round, out_name): extra = (f"--rows {test_f} --out {results_dir}/{out_name} --gpus {args.gpus}") if args.no_tta: extra += " --no-tta" - py("05_nnu_eval_test.py", extra) + py("08_eval.py", extra) return os.path.join(results_dir, out_name) + def build_table(): + # Rewrite the per-round table + plot; called after every round's eval so + # each round's performance is saved incrementally (survives interruptions). + table = [] + for k in range(args.rounds + 1): + resf = os.path.join(results_dir, f"round{k}_test_nnu.json") + if not os.path.exists(resf): + continue + r = json.load(open(resf)) + row = {"round": k, "test_dice": round(r["dice"], 4), "test_dice_hard": round(r["dice_hard"], 4), + "n_test": r["n"], "ckpt": os.path.join(ROOT, "runs_nnu", f"round{k}", "best_nnu.pth")} + pf = os.path.join(pdir_root, f"round{k}", "summary.json") + if k > 0 and os.path.exists(pf): + s = json.load(open(pf)) + row.update({"n_pos": s["n_pos"], "n_neg": s["n_neg"], + "n_rej_cons": s["n_rejected_consistency"], "n_error": s["n_error"], + "pos_vol_med_mm3": s["pos_vol_mm3"]["med"]}) + table.append(row) + save_jsonl(table, os.path.join(results_dir, "iterative_table_nnu.jsonl")) + if not table: + return table + print(json.dumps(table, indent=1)) + try: + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + rs = [t["round"] for t in table] + ds = [t["test_dice"] for t in table] + plt.figure(figsize=(6, 4)) + plt.plot(rs, ds, "o-") + plt.xlabel("pseudo-labeling round") + plt.ylabel("holdout tumor Dice (nnU-Net)") + for x, y in zip(rs, ds): + plt.annotate(f"{y:.3f}", (x, y), textcoords="offset points", xytext=(0, 8), fontsize=8) + plt.grid(alpha=0.3) + plt.tight_layout() + plt.savefig(os.path.join(results_dir, "iterative_dice_nnu.png"), dpi=150) + except Exception as e: # noqa + print("plot failed:", repr(e)) + return table + # ---- round 0: baseline ---- prepare(base) plan() @@ -112,6 +153,7 @@ def main(): accepted_f = [predict_pool(1, None)] save_jsonl(load_jsonl(accepted_f[0]), added_f) eval_test(0, "round0_test_nnu.json") + build_table() for k in range(1, args.rounds + 1): rows = list(base) @@ -127,42 +169,10 @@ def main(): accepted_f.append(predict_pool(k + 1, added_f)) save_jsonl([r for f in accepted_f for r in load_jsonl(f)], added_f) eval_test(k, f"round{k}_test_nnu.json") + build_table() # ---- report (same layout as scripts/09) ---- - table = [] - for k in range(args.rounds + 1): - resf = os.path.join(results_dir, f"round{k}_test_nnu.json") - if not os.path.exists(resf): - continue - r = json.load(open(resf)) - row = {"round": k, "test_dice": round(r["dice"], 4), "test_dice_hard": round(r["dice_hard"], 4), - "n_test": r["n"], "ckpt": os.path.join(ROOT, "runs_nnu", f"round{k}", "best_nnu.pth")} - pf = os.path.join(pdir_root, f"round{k}", "summary.json") - if k > 0 and os.path.exists(pf): - s = json.load(open(pf)) - row.update({"n_pos": s["n_pos"], "n_neg": s["n_neg"], - "n_rej_cons": s["n_rejected_consistency"], "n_error": s["n_error"], - "pos_vol_med_mm3": s["pos_vol_mm3"]["med"]}) - table.append(row) - save_jsonl(table, os.path.join(results_dir, "iterative_table_nnu.jsonl")) - print(json.dumps(table, indent=1)) - try: - import matplotlib - matplotlib.use("Agg") - import matplotlib.pyplot as plt - rs = [t["round"] for t in table] - ds = [t["test_dice"] for t in table] - plt.figure(figsize=(6, 4)) - plt.plot(rs, ds, "o-") - plt.xlabel("pseudo-labeling round") - plt.ylabel("holdout tumor Dice (nnU-Net)") - for x, y in zip(rs, ds): - plt.annotate(f"{y:.3f}", (x, y), textcoords="offset points", xytext=(0, 8), fontsize=8) - plt.grid(alpha=0.3) - plt.tight_layout() - plt.savefig(os.path.join(results_dir, "iterative_dice_nnu.png"), dpi=150) - except Exception as e: # noqa - print("plot failed:", repr(e)) + build_table() if __name__ == "__main__":