feat(benchmark): support multi-dataset evaluation with configurable top-k list

- Evaluate multiple datasets in a single run (CIFAR10 + CIFAR100)
- Report Recall@K for a list of K values from one underlying search
- Save results to disk: summary.md, metrics.csv, per-dataset predictions.csv, confusion_matrix.csv
- Richer evaluation output: query_labels, topk_ids, topk_labels for downstream analysis
- Add --dataset and --top-k CLI overrides for benchmark command
- Update config schema: dataset→datasets, top_k→top_k_list
This commit is contained in:
2026-05-31 15:56:35 +08:00
parent 7a1e1ccf3f
commit 9eb52f8cef
6 changed files with 454 additions and 96 deletions

View File

@@ -22,14 +22,18 @@ dataset:
seed: 42
benchmark:
dataset:
source_type: "huggingface"
path: "uoft-cs/cifar100"
img_column: "img"
label_column: "fine_label"
datasets:
- source_type: "huggingface"
path: "uoft-cs/cifar100"
img_column: "img"
label_column: "fine_label"
- source_type: "huggingface"
path: "uoft-cs/cifar10"
img_column: "img"
label_column: "label"
task:
name: "recall_at_k"
type: "retrieval"
top_k: 1
top_k_list: [1, 5, 10]
batch_size: 64
model_table_prefix: "benchmark"