feat(compressors): add JSONL training metrics logging with CLI controls

- Add write_training_metrics() in new compressors/training_metrics.py
  for appending epoch/step/lr/component rows as JSON Lines
- Wire --metrics-path and --log-every CLI options into train.py, passing
  them to the training loop so metrics rows are written every N steps
- Accept absolute metrics paths or paths relative to output directory
- Add quantization component to loss log alongside existing distill/contrastive
- Replace inline torch.device() with get_device() utility
- Add test_hash_training_metrics.py covering multi-row JSONL append

Infrastructure:
- Pin torch 2.7.1 + CUDA 12.8 index for Linux/Windows in pyproject.toml
- Add .justfile rsync upload recipe with .stignore exclusion
- Exclude **/__marimo__ from rsync in .stignore

Dependencies updated: numpy 2.4.5, pandas 3.0.3, black 26.5.0,
click 8.4.0, contourpy, etc.
This commit is contained in:
2026-05-17 14:18:51 +08:00
parent e8c890a69f
commit 4ea567adba
8 changed files with 864 additions and 531 deletions

View File

@@ -0,0 +1,61 @@
import json
import sys
from pathlib import Path
COMPRESSORS_DIR = Path(__file__).resolve().parents[1] / "mini-nav" / "compressors"
sys.path.insert(0, str(COMPRESSORS_DIR))
from training_metrics import write_training_metrics # noqa: E402
def test_write_training_metrics_appends_jsonl_rows(tmp_path):
metrics_path = tmp_path / "hash_training_metrics.jsonl"
write_training_metrics(
metrics_path,
epoch=2,
step=17,
lr=1e-4,
components={
"total": 1.25,
"contrastive": 0.75,
"distill": 0.4,
"quantization": 0.1,
},
)
write_training_metrics(
metrics_path,
epoch=2,
step=18,
lr=1e-4,
components={
"total": 1.0,
"contrastive": 0.6,
"distill": 0.3,
"quantization": 0.1,
},
)
rows = [json.loads(line) for line in metrics_path.read_text().splitlines()]
assert rows == [
{
"epoch": 2,
"step": 17,
"lr": 1e-4,
"total": 1.25,
"contrastive": 0.75,
"distill": 0.4,
"quantization": 0.1,
},
{
"epoch": 2,
"step": 18,
"lr": 1e-4,
"total": 1.0,
"contrastive": 0.6,
"distill": 0.3,
"quantization": 0.1,
},
]