mirror of
https://github.com/SikongJueluo/Mini-Nav.git
synced 2026-07-12 20:15:31 +08:00
SikongJueluo
97e53d44f8
Add explicit separation between query-only cycles (accept→last) and end-to-end cycles (load + write + noise + queries) in hardware retrieval benchmarks. - Add query_only_cycles_per_query, load_write_noise_cycles, end_to_end_cycles metrics - Refactor summarize_query_timings() to use accept_to_last_result_cycles as query-only base - Add build_hardware_performance() to compute end-to-end performance separately - Add current_sim_cycle() helper using cocotb get_sim_time - Update CSV/Markdown outputs and RETRIEVAL_PERF_RESULT log format - Update documentation to clarify cycle-counting methodology - Update tests to cover new performance measurement logic
524 lines
21 KiB
Python
524 lines
21 KiB
Python
from __future__ import annotations
|
|
|
|
import csv
|
|
import json
|
|
import os
|
|
from dataclasses import dataclass
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
import cocotb
|
|
import numpy as np
|
|
from cocotb.clock import Clock
|
|
from cocotb.utils import get_sim_time
|
|
|
|
from model.ref_model import (
|
|
match_topk,
|
|
match_topk_from_scores,
|
|
)
|
|
from tests.top.utils import (
|
|
dut_hash_bits,
|
|
dut_lanes,
|
|
dut_num_rows,
|
|
get_param,
|
|
query_topk_once_with_latency,
|
|
reset_dut,
|
|
write_rows,
|
|
)
|
|
|
|
MAX_BENCHMARK_QUERIES = 128
|
|
DEFAULT_POSITIVES_PER_CLASS = 8
|
|
DEFAULT_QUERIES_PER_CLASS = 2
|
|
DEFAULT_ROW_FLIP_BITS = 16
|
|
DEFAULT_QUERY_FLIP_BITS = 16
|
|
DEFAULT_SEED = 20260522
|
|
BENCHMARK_KS = (1, 5)
|
|
SIM_CLOCK_PERIOD_NS = 10
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class RetrievalDataset:
|
|
rows: list[int]
|
|
row_labels: list[int]
|
|
queries: list[int]
|
|
query_labels: list[int]
|
|
num_classes: int
|
|
positives_per_class: int
|
|
queries_per_class: int
|
|
seed: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class QueryTiming:
|
|
accept_to_first_result_cycles: int
|
|
accept_to_last_result_cycles: int
|
|
total_query_cycles: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class MetricAccumulator:
|
|
precision_sum: float = 0.0
|
|
recall_sum: float = 0.0
|
|
f1_sum: float = 0.0
|
|
exact_matches: int = 0
|
|
label_hits: int = 0
|
|
count: int = 0
|
|
|
|
def add(self, precision: float, recall: float, f1: float, label_hit: bool, exact: bool) -> "MetricAccumulator":
|
|
return MetricAccumulator(
|
|
precision_sum=self.precision_sum + precision,
|
|
recall_sum=self.recall_sum + recall,
|
|
f1_sum=self.f1_sum + f1,
|
|
exact_matches=self.exact_matches + int(exact),
|
|
label_hits=self.label_hits + int(label_hit),
|
|
count=self.count + 1,
|
|
)
|
|
|
|
def as_dict(self) -> dict[str, float]:
|
|
if self.count == 0:
|
|
return {
|
|
"macro_precision": 0.0,
|
|
"retrieval_recall": 0.0,
|
|
"macro_f1": 0.0,
|
|
"exact_match_rate": 0.0,
|
|
"recall@k": 0.0,
|
|
}
|
|
return {
|
|
"macro_precision": self.precision_sum / self.count,
|
|
"retrieval_recall": self.recall_sum / self.count,
|
|
"macro_f1": self.f1_sum / self.count,
|
|
"exact_match_rate": self.exact_matches / self.count,
|
|
"recall@k": self.label_hits / self.count,
|
|
}
|
|
|
|
|
|
def _project_root() -> Path:
|
|
return Path(__file__).resolve().parents[4]
|
|
|
|
|
|
def _flip_exact_bits(rng: np.random.Generator, width: int, n_bits: int) -> int:
|
|
n_bits = max(0, min(int(n_bits), int(width)))
|
|
if n_bits == 0:
|
|
return 0
|
|
positions = rng.choice(width, size=n_bits, replace=False)
|
|
mask = 0
|
|
for pos in positions:
|
|
mask |= 1 << int(pos)
|
|
return mask
|
|
|
|
|
|
def words_le_to_int(words: np.ndarray) -> int:
|
|
value = 0
|
|
for idx, word in enumerate(words.tolist()):
|
|
value |= int(word) << (64 * idx)
|
|
return value
|
|
|
|
|
|
def load_retrieval_dataset_npz(path: str | os.PathLike[str]) -> RetrievalDataset:
|
|
dataset_path = Path(path)
|
|
if not dataset_path.is_absolute():
|
|
dataset_path = _project_root() / dataset_path
|
|
if not dataset_path.exists():
|
|
raise AssertionError(f"CAM_RETRIEVAL_DATASET not found: {dataset_path}")
|
|
loaded = np.load(dataset_path)
|
|
rows = [words_le_to_int(words) for words in loaded["rows_words"]]
|
|
queries = [words_le_to_int(words) for words in loaded["queries_words"]]
|
|
row_labels = [int(x) for x in loaded["row_labels"].tolist()]
|
|
query_labels = [int(x) for x in loaded["query_labels"].tolist()]
|
|
|
|
return RetrievalDataset(
|
|
rows=rows,
|
|
row_labels=row_labels,
|
|
queries=queries,
|
|
query_labels=query_labels,
|
|
num_classes=len(set(row_labels)),
|
|
positives_per_class=0,
|
|
queries_per_class=0,
|
|
seed=0,
|
|
)
|
|
|
|
|
|
def make_clustered_dataset(
|
|
*,
|
|
num_rows: int,
|
|
hash_bits: int,
|
|
positives_per_class: int = DEFAULT_POSITIVES_PER_CLASS,
|
|
queries_per_class: int = DEFAULT_QUERIES_PER_CLASS,
|
|
row_flip_bits: int = DEFAULT_ROW_FLIP_BITS,
|
|
query_flip_bits: int = DEFAULT_QUERY_FLIP_BITS,
|
|
seed: int = DEFAULT_SEED,
|
|
) -> RetrievalDataset:
|
|
usable_rows = int(num_rows)
|
|
if usable_rows < 5:
|
|
raise AssertionError("Retrieval benchmark requires at least 5 CAM rows")
|
|
|
|
positives_per_class = min(positives_per_class, usable_rows)
|
|
num_classes = max(1, usable_rows // positives_per_class)
|
|
usable_rows = num_classes * positives_per_class
|
|
|
|
# Cap total queries to keep simulation runtime bounded
|
|
max_queries = min(MAX_BENCHMARK_QUERIES, num_classes * queries_per_class)
|
|
if max_queries < num_classes * queries_per_class:
|
|
queries_per_class = max(1, max_queries // num_classes)
|
|
|
|
rng = np.random.default_rng(seed)
|
|
mask = (1 << hash_bits) - 1
|
|
words = (hash_bits + 63) // 64
|
|
|
|
rows: list[int] = []
|
|
row_labels: list[int] = []
|
|
queries: list[int] = []
|
|
query_labels: list[int] = []
|
|
|
|
for class_id in range(num_classes):
|
|
center = 0
|
|
for word in range(words):
|
|
center |= int(rng.integers(0, 1 << 64, dtype=np.uint64)) << (64 * word)
|
|
center &= mask
|
|
|
|
for _ in range(positives_per_class):
|
|
rows.append((center ^ _flip_exact_bits(rng, hash_bits, row_flip_bits)) & mask)
|
|
row_labels.append(class_id)
|
|
|
|
for _ in range(queries_per_class):
|
|
queries.append((center ^ _flip_exact_bits(rng, hash_bits, query_flip_bits)) & mask)
|
|
query_labels.append(class_id)
|
|
|
|
return RetrievalDataset(
|
|
rows=rows,
|
|
row_labels=row_labels,
|
|
queries=queries,
|
|
query_labels=query_labels,
|
|
num_classes=num_classes,
|
|
positives_per_class=positives_per_class,
|
|
queries_per_class=queries_per_class,
|
|
seed=seed,
|
|
)
|
|
|
|
|
|
def compute_metrics(topk_indices: list[int], row_labels: list[int], query_label: int, k: int) -> tuple[float, float, float]:
|
|
retrieved = topk_indices[:k]
|
|
relevant = {idx for idx, label in enumerate(row_labels) if label == query_label}
|
|
tp = len(set(retrieved) & relevant)
|
|
precision = tp / float(k)
|
|
recall = tp / float(len(relevant)) if relevant else 0.0
|
|
f1 = 0.0 if precision + recall == 0 else (2.0 * precision * recall) / (precision + recall)
|
|
return precision, recall, f1
|
|
|
|
|
|
def mode_from_params(write_noise_en: int) -> str:
|
|
if write_noise_en:
|
|
return "write_noise"
|
|
return "no_noise"
|
|
|
|
|
|
def summarize_query_timings(timings: list[QueryTiming]) -> dict[str, float]:
|
|
if not timings:
|
|
return {
|
|
"num_queries": 0,
|
|
"query_only_total_cycles": 0,
|
|
"query_only_cycles_per_query": 0.0,
|
|
"query_only_min_cycles": 0,
|
|
"query_only_max_cycles": 0,
|
|
"query_only_queries_per_cycle": 0.0,
|
|
"total_query_cycles": 0,
|
|
"mean_total_query_cycles": 0.0,
|
|
"min_total_query_cycles": 0,
|
|
"max_total_query_cycles": 0,
|
|
"mean_accept_to_first_result_cycles": 0.0,
|
|
"mean_accept_to_last_result_cycles": 0.0,
|
|
"cycles_per_query": 0.0,
|
|
"queries_per_cycle": 0.0,
|
|
}
|
|
|
|
total_cycles = sum(t.accept_to_last_result_cycles for t in timings)
|
|
total_first = sum(t.accept_to_first_result_cycles for t in timings)
|
|
total_last = sum(t.accept_to_last_result_cycles for t in timings)
|
|
count = len(timings)
|
|
mean_last = total_last / float(count)
|
|
queries_per_cycle = count / float(total_cycles) if total_cycles > 0 else 0.0
|
|
return {
|
|
"num_queries": count,
|
|
"query_only_total_cycles": total_cycles,
|
|
"query_only_cycles_per_query": mean_last,
|
|
"query_only_min_cycles": min(t.accept_to_last_result_cycles for t in timings),
|
|
"query_only_max_cycles": max(t.accept_to_last_result_cycles for t in timings),
|
|
"query_only_queries_per_cycle": queries_per_cycle,
|
|
# Backward-compatible aliases: query-only, not end-to-end.
|
|
"total_query_cycles": total_cycles,
|
|
"mean_total_query_cycles": total_cycles / float(count),
|
|
"min_total_query_cycles": min(t.accept_to_last_result_cycles for t in timings),
|
|
"max_total_query_cycles": max(t.accept_to_last_result_cycles for t in timings),
|
|
"mean_accept_to_first_result_cycles": total_first / float(count),
|
|
"mean_accept_to_last_result_cycles": mean_last,
|
|
"cycles_per_query": mean_last,
|
|
"queries_per_cycle": queries_per_cycle,
|
|
}
|
|
|
|
|
|
def build_hardware_performance(
|
|
timings: list[QueryTiming],
|
|
*,
|
|
load_write_noise_cycles: int,
|
|
end_to_end_cycles: int,
|
|
) -> dict[str, float]:
|
|
performance = summarize_query_timings(timings)
|
|
num_queries = int(performance["num_queries"])
|
|
performance.update({
|
|
"load_write_noise_cycles": int(load_write_noise_cycles),
|
|
"end_to_end_cycles": int(end_to_end_cycles),
|
|
"end_to_end_cycles_per_query": (
|
|
float(end_to_end_cycles) / float(num_queries) if num_queries > 0 else 0.0
|
|
),
|
|
"end_to_end_queries_per_cycle": (
|
|
float(num_queries) / float(end_to_end_cycles) if end_to_end_cycles > 0 else 0.0
|
|
),
|
|
})
|
|
return performance
|
|
|
|
|
|
def current_sim_cycle() -> int:
|
|
"""Return the current benchmark clock cycle from simulator time."""
|
|
return int(get_sim_time("ns") // SIM_CLOCK_PERIOD_NS)
|
|
|
|
|
|
def output_dir_for(mode: str) -> Path:
|
|
run_id = os.environ.get("CAM_RETRIEVAL_RUN_ID")
|
|
if not run_id:
|
|
run_id = f"{datetime.now().strftime('%Y-%m-%d-%H%M%S')}-{mode}"
|
|
out_dir = _project_root() / "outputs" / "cam_retrieval_benchmark" / run_id
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
(out_dir / "logs").mkdir(exist_ok=True)
|
|
return out_dir
|
|
|
|
|
|
def write_outputs(out_dir: Path, result: dict) -> None:
|
|
metrics_json = out_dir / "metrics.json"
|
|
metrics_csv = out_dir / "metrics.csv"
|
|
summary_md = out_dir / "summary.md"
|
|
|
|
metrics_json.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
|
|
|
fieldnames = [
|
|
"run_id", "mode", "num_rows", "hash_bits", "lanes", "topk_k",
|
|
"write_noise_en", "write_noise_rate_num",
|
|
"write_noise_rate_den",
|
|
"num_queries", "k", "macro_precision", "retrieval_recall", "macro_f1",
|
|
"recall@k", "exact_match_rate",
|
|
"query_only_cycles_per_query", "query_only_total_cycles",
|
|
"query_only_queries_per_cycle", "load_write_noise_cycles",
|
|
"end_to_end_cycles", "end_to_end_cycles_per_query",
|
|
"end_to_end_queries_per_cycle", "cycles_per_query",
|
|
"mean_accept_to_first_result_cycles", "mean_accept_to_last_result_cycles",
|
|
"mean_total_query_cycles", "total_query_cycles", "queries_per_cycle", "status",
|
|
]
|
|
with metrics_csv.open("w", newline="", encoding="utf-8") as f:
|
|
writer = csv.DictWriter(f, fieldnames=fieldnames)
|
|
writer.writeheader()
|
|
for k, metrics in result["metrics"].items():
|
|
row = {
|
|
"run_id": result["run_id"],
|
|
"mode": result["mode"],
|
|
"num_rows": result["params"]["num_rows"],
|
|
"hash_bits": result["params"]["hash_bits"],
|
|
"lanes": result["params"]["lanes"],
|
|
"topk_k": result["params"]["topk_k"],
|
|
"write_noise_en": result["params"]["write_noise_en"],
|
|
"write_noise_rate_num": result["params"]["write_noise_rate_num"],
|
|
"write_noise_rate_den": result["params"]["write_noise_rate_den"],
|
|
"num_queries": result["dataset"]["num_queries"],
|
|
"k": int(k),
|
|
"macro_precision": metrics["macro_precision"],
|
|
"retrieval_recall": metrics["retrieval_recall"],
|
|
"macro_f1": metrics["macro_f1"],
|
|
"recall@k": metrics["recall@k"],
|
|
"exact_match_rate": metrics["exact_match_rate"],
|
|
"query_only_cycles_per_query": result.get("performance", {}).get("query_only_cycles_per_query", ""),
|
|
"query_only_total_cycles": result.get("performance", {}).get("query_only_total_cycles", ""),
|
|
"query_only_queries_per_cycle": result.get("performance", {}).get("query_only_queries_per_cycle", ""),
|
|
"load_write_noise_cycles": result.get("performance", {}).get("load_write_noise_cycles", ""),
|
|
"end_to_end_cycles": result.get("performance", {}).get("end_to_end_cycles", ""),
|
|
"end_to_end_cycles_per_query": result.get("performance", {}).get("end_to_end_cycles_per_query", ""),
|
|
"end_to_end_queries_per_cycle": result.get("performance", {}).get("end_to_end_queries_per_cycle", ""),
|
|
"cycles_per_query": result.get("performance", {}).get("cycles_per_query", ""),
|
|
"mean_accept_to_first_result_cycles": result.get("performance", {}).get(
|
|
"mean_accept_to_first_result_cycles", "",
|
|
),
|
|
"mean_accept_to_last_result_cycles": result.get("performance", {}).get(
|
|
"mean_accept_to_last_result_cycles", "",
|
|
),
|
|
"mean_total_query_cycles": result.get("performance", {}).get("mean_total_query_cycles", ""),
|
|
"total_query_cycles": result.get("performance", {}).get("total_query_cycles", ""),
|
|
"queries_per_cycle": result.get("performance", {}).get("queries_per_cycle", ""),
|
|
"status": result["status"],
|
|
}
|
|
writer.writerow(row)
|
|
|
|
lines = [
|
|
"# CAM Retrieval Benchmark Summary",
|
|
"",
|
|
f"- run_id: `{result['run_id']}`",
|
|
f"- mode: `{result['mode']}`",
|
|
f"- status: `{result['status']}`",
|
|
f"- num_queries: `{result['dataset']['num_queries']}`",
|
|
"",
|
|
"## Hardware performance",
|
|
"",
|
|
f"- query-only cycles/query: `{result.get('performance', {}).get('query_only_cycles_per_query', '')}`",
|
|
f"- query-only total cycles: `{result.get('performance', {}).get('query_only_total_cycles', '')}`",
|
|
f"- query-only queries/cycle: `{result.get('performance', {}).get('query_only_queries_per_cycle', '')}`",
|
|
f"- load/write/noise cycles: `{result.get('performance', {}).get('load_write_noise_cycles', '')}`",
|
|
f"- end-to-end cycles: `{result.get('performance', {}).get('end_to_end_cycles', '')}`",
|
|
f"- end-to-end cycles/query: `{result.get('performance', {}).get('end_to_end_cycles_per_query', '')}`",
|
|
f"- end-to-end queries/cycle: `{result.get('performance', {}).get('end_to_end_queries_per_cycle', '')}`",
|
|
f"- cycles_per_query (compat, query-only): `{result.get('performance', {}).get('cycles_per_query', '')}`",
|
|
f"- accept_to_first_result_cycles: `{result.get('performance', {}).get('mean_accept_to_first_result_cycles', '')}`",
|
|
f"- accept_to_last_result_cycles: `{result.get('performance', {}).get('mean_accept_to_last_result_cycles', '')}`",
|
|
f"- total_query_cycles (compat, query-only): `{result.get('performance', {}).get('total_query_cycles', '')}`",
|
|
f"- queries_per_cycle (compat, query-only): `{result.get('performance', {}).get('queries_per_cycle', '')}`",
|
|
"",
|
|
"## Retrieval quality",
|
|
"",
|
|
"| k | macro_precision | retrieval_recall | macro_f1 | recall@k | exact_match_rate |",
|
|
"|---:|---:|---:|---:|---:|---:|",
|
|
]
|
|
for k, metrics in result["metrics"].items():
|
|
lines.append(
|
|
f"| {k} | {metrics['macro_precision']:.6f} | {metrics['retrieval_recall']:.6f} | "
|
|
f"{metrics['macro_f1']:.6f} | {metrics['recall@k']:.6f} | {metrics['exact_match_rate']:.6f} |"
|
|
)
|
|
lines.extend([
|
|
"",
|
|
"说明：结果来自 Verilator/Cocotb 仿真，不是 FPGA 板上实测。",
|
|
])
|
|
summary_md.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
|
|
|
|
@cocotb.test()
|
|
async def cam_retrieval_benchmark(dut):
|
|
cocotb.start_soon(Clock(dut.clk, SIM_CLOCK_PERIOD_NS, unit="ns").start())
|
|
await reset_dut(dut)
|
|
|
|
num_rows = dut_num_rows(dut)
|
|
hash_bits = dut_hash_bits(dut)
|
|
lanes = dut_lanes(dut)
|
|
write_noise_en = int(get_param(dut, "WRITE_NOISE_EN", 0) or 0)
|
|
write_noise_rate_num = int(get_param(dut, "WRITE_NOISE_RATE_NUM", 0) or 0)
|
|
write_noise_rate_den = int(get_param(dut, "WRITE_NOISE_RATE_DEN", 100) or 100)
|
|
mode = mode_from_params(write_noise_en)
|
|
|
|
if num_rows % lanes != 0:
|
|
raise AssertionError("Retrieval benchmark requires NUM_ROWS divisible by LANES")
|
|
|
|
dataset_path = os.environ.get("CAM_RETRIEVAL_DATASET")
|
|
if not dataset_path:
|
|
raise AssertionError("CAM_RETRIEVAL_DATASET is required; run scripts/prepare_cam_retrieval_dataset.py first")
|
|
dataset = load_retrieval_dataset_npz(dataset_path)
|
|
if len(dataset.rows) != num_rows:
|
|
raise AssertionError(f"artifact row count {len(dataset.rows)} must equal DUT NUM_ROWS {num_rows}")
|
|
|
|
benchmark_start_cycle = current_sim_cycle()
|
|
load_write_noise_start_cycle = current_sim_cycle()
|
|
await write_rows(dut, dataset.rows)
|
|
load_write_noise_cycles = current_sim_cycle() - load_write_noise_start_cycle
|
|
|
|
accumulators = {k: MetricAccumulator() for k in BENCHMARK_KS}
|
|
timings: list[QueryTiming] = []
|
|
|
|
for query, query_label in zip(dataset.queries, dataset.query_labels):
|
|
beats, _, _, _, timing = await query_topk_once_with_latency(dut, query)
|
|
timings.append(QueryTiming(**timing))
|
|
if len(beats) < max(BENCHMARK_KS):
|
|
raise AssertionError(f"Expected at least {max(BENCHMARK_KS)} Top-K beats, got {len(beats)}")
|
|
|
|
dut_topk = [int(beat[1]) for beat in beats[: max(BENCHMARK_KS)]]
|
|
|
|
golden_topk, _ = match_topk(query, dataset.rows, width=hash_bits, k=max(BENCHMARK_KS))
|
|
|
|
for k in BENCHMARK_KS:
|
|
precision, recall, f1 = compute_metrics(dut_topk, dataset.row_labels, query_label, k)
|
|
exact = dut_topk[:k] == golden_topk[:k]
|
|
retrieved_labels = [dataset.row_labels[idx] for idx in dut_topk[:k]]
|
|
label_hit = query_label in retrieved_labels
|
|
accumulators[k] = accumulators[k].add(precision, recall, f1, label_hit, exact)
|
|
|
|
end_to_end_cycles = current_sim_cycle() - benchmark_start_cycle
|
|
|
|
run_id = os.environ.get("CAM_RETRIEVAL_RUN_ID") or f"{datetime.now().strftime('%Y-%m-%d-%H%M%S')}-{mode}"
|
|
result = {
|
|
"run_id": run_id,
|
|
"mode": mode,
|
|
"status": "pass",
|
|
"params": {
|
|
"num_rows": len(dataset.rows),
|
|
"hash_bits": hash_bits,
|
|
"lanes": lanes,
|
|
"topk_k": max(BENCHMARK_KS),
|
|
"write_noise_en": write_noise_en,
|
|
"write_noise_rate_num": write_noise_rate_num,
|
|
"write_noise_rate_den": write_noise_rate_den,
|
|
},
|
|
"dataset": {
|
|
"num_classes": dataset.num_classes,
|
|
"positives_per_class": dataset.positives_per_class,
|
|
"queries_per_class": dataset.queries_per_class,
|
|
"num_queries": len(dataset.queries),
|
|
"seed": dataset.seed,
|
|
},
|
|
"metrics": {str(k): accumulators[k].as_dict() for k in BENCHMARK_KS},
|
|
"performance": build_hardware_performance(
|
|
timings,
|
|
load_write_noise_cycles=load_write_noise_cycles,
|
|
end_to_end_cycles=end_to_end_cycles,
|
|
),
|
|
}
|
|
|
|
out_dir = output_dir_for(mode)
|
|
write_outputs(out_dir, result)
|
|
|
|
for k in BENCHMARK_KS:
|
|
metrics = result["metrics"][str(k)]
|
|
dut._log.info(
|
|
"RETRIEVAL_RESULT mode=%s k=%d precision=%.6f retrieval_recall=%.6f f1=%.6f recall_at_k=%.6f exact_match=%.6f output_dir=%s",
|
|
mode, k, metrics["macro_precision"], metrics["retrieval_recall"],
|
|
metrics["macro_f1"], metrics["recall@k"], metrics["exact_match_rate"],
|
|
str(out_dir.relative_to(_project_root())),
|
|
)
|
|
|
|
performance = result["performance"]
|
|
dut._log.info(
|
|
"RETRIEVAL_PERF_RESULT mode=%s num_queries=%d query_only_cycles_per_query=%.6f "
|
|
"query_only_total_cycles=%d query_only_queries_per_cycle=%.9f "
|
|
"load_write_noise_cycles=%d end_to_end_cycles=%d "
|
|
"end_to_end_cycles_per_query=%.6f end_to_end_queries_per_cycle=%.9f "
|
|
"cycles_per_query=%.6f "
|
|
"accept_to_first_result_cycles=%.6f accept_to_last_result_cycles=%.6f "
|
|
"total_query_cycles=%d queries_per_cycle=%.9f status=pass output_dir=%s",
|
|
mode,
|
|
performance["num_queries"],
|
|
performance["query_only_cycles_per_query"],
|
|
performance["query_only_total_cycles"],
|
|
performance["query_only_queries_per_cycle"],
|
|
performance["load_write_noise_cycles"],
|
|
performance["end_to_end_cycles"],
|
|
performance["end_to_end_cycles_per_query"],
|
|
performance["end_to_end_queries_per_cycle"],
|
|
performance["cycles_per_query"],
|
|
performance["mean_accept_to_first_result_cycles"],
|
|
performance["mean_accept_to_last_result_cycles"],
|
|
performance["total_query_cycles"],
|
|
performance["queries_per_cycle"],
|
|
str(out_dir.relative_to(_project_root())),
|
|
)
|
|
|
|
if write_noise_en == 0:
|
|
assert result["metrics"]["5"]["exact_match_rate"] == 1.0, (
|
|
f"Expected perfect exact match with no noise, got "
|
|
f"{result['metrics']['5']['exact_match_rate']}"
|
|
)
|
|
else:
|
|
dut._log.info(
|
|
"Noise enabled (WRITE_NOISE_RATE=%d/%d) — exact_match assertion skipped",
|
|
write_noise_rate_num, write_noise_rate_den,
|
|
)
|