diff --git a/hw/sim/Makefile b/hw/sim/Makefile index 489f0ba..0156073 100644 --- a/hw/sim/Makefile +++ b/hw/sim/Makefile @@ -27,6 +27,18 @@ READ_NOISE_RATE_NUM ?= READ_NOISE_RATE_DEN ?= READ_NOISE_BITS ?= +# Perf-test compatibility override: when running cam_perf, force deterministic +# noise parameters so noise_mask_grouped.sv elaborates correctly for perf +# sweep HASH_BITS values (64, 128, 192, 256, 320, 384, 448, 512 — all +# divisible by 64 such that HASH_BITS / NOISE_BITS == 64 in the RTL). +# This is a testbench parameter, not a hardware performance/resource claim. +ifeq ($(COCOTB_TEST_MODULES),tests.test_cam_perf) +WRITE_NOISE_EN := 0 +READ_NOISE_EN := 0 +WRITE_NOISE_BITS := $(shell echo $$(( $(HASH_BITS) / 64 ))) +READ_NOISE_BITS := $(shell echo $$(( $(HASH_BITS) / 64 ))) +endif + ifneq ($(strip $(WRITE_NOISE_EN)),) COMPILE_ARGS += -GWRITE_NOISE_EN=$(WRITE_NOISE_EN) endif diff --git a/hw/sim/tests/test_cam_perf.py b/hw/sim/tests/test_cam_perf.py new file mode 100644 index 0000000..5eaba87 --- /dev/null +++ b/hw/sim/tests/test_cam_perf.py @@ -0,0 +1,306 @@ +from __future__ import annotations + +import cocotb +from cocotb.clock import Clock +from cocotb.triggers import RisingEdge, ReadOnly, Timer + + +# ── Helpers (local reimplementation — no imports from test_cam_basic) ───────── + + +def _get_param(dut, name, default=None): + """Read a Verilator-exposed parameter from the DUT.""" + try: + val = getattr(dut, name, None) + if val is not None: + return int(val.value) + except Exception: + pass + return default + + +def dut_num_rows(dut): + val = _get_param(dut, "NUM_ROWS", None) + if val is not None: + return val + return 1 << len(dut.wr_addr) + + +def dut_hash_bits(dut): + val = _get_param(dut, "HASH_BITS", None) + if val is not None: + return val + return len(dut.write_hash) + + +def dut_lanes(dut): + val = _get_param(dut, "LANES", None) + if val is not None: + return val + return len(dut.rd_resp_row_ids) // len(dut.wr_addr) + + +def _pipeline_depth(num_rows, lanes): + """Number of match-engine pipeline stages (rows processed per query).""" + return num_rows // lanes + + +# --------------------------------------------------------------------------- +# Bounded wait helper (RisingEdge only — keep the caller writeable) +# --------------------------------------------------------------------------- + + +async def _wait_bounded(dut, condition_fn, max_cycles, label): + """Wait for *condition_fn* to become ``True``, sampling at each RisingEdge. + + Raises ``AssertionError`` with *label* if *max_cycles* is exceeded. + The caller is left in a writeable phase after return. + """ + for _ in range(max_cycles): + await RisingEdge(dut.clk) + if condition_fn(): + return + raise AssertionError( + f"{label}: timed out after {max_cycles} clock cycles" + ) + + +def _condition_signal_high(signal): + """Return a nullary callable that checks *signal* is high (integer 1).""" + def _check(): + return int(signal.value) == 1 + return _check + + +# --------------------------------------------------------------------------- +# DUT helpers +# --------------------------------------------------------------------------- + + +async def reset_dut(dut): + """Reset the DUT with new handshake interface. + + ``result_ready`` is held at 0 during measurement; the performance test + pulses it high for exactly one cycle after capturing ``result_valid``. + """ + dut.rst_n.value = 0 + dut.wr_valid.value = 0 + dut.wr_addr.value = 0 + dut.write_hash.value = 0 + dut.query_valid.value = 0 + dut.query_hash.value = 0 + dut.result_ready.value = 0 + + for _ in range(5): + await RisingEdge(dut.clk) + + dut.rst_n.value = 1 + + for _ in range(2): + await RisingEdge(dut.clk) + + +async def wait_idle(dut, max_cycles=10): + """Wait until both wr_ready=1 and query_ready=1 (system fully idle).""" + await _wait_bounded( + dut, + lambda: int(dut.wr_ready.value) and int(dut.query_ready.value), + max_cycles, + "wait_idle", + ) + + +async def write_row(dut, addr, value, max_cycles=10): + """Write a single row using wr_valid/wr_ready handshake.""" + await wait_idle(dut) + + dut.wr_addr.value = addr + dut.write_hash.value = int(value) + dut.wr_valid.value = 1 + + await _wait_bounded( + dut, + _condition_signal_high(dut.wr_ready), + max_cycles, + f"write_row(addr={addr}) handshake", + ) + + dut.wr_valid.value = 0 + await wait_idle(dut) + + +async def write_rows(dut, rows): + """Write all rows sequentially.""" + for idx, value in enumerate(rows): + await write_row(dut, idx, value) + + +async def query_once_with_latency(dut, query, max_result_cycles): + """Issue a query and return (top1_index, top1_score, latency_cycles, total_cycles). + + Parameters + ---------- + max_result_cycles : int + Hard bound on cycles from query acceptance to *result_valid*. + Derived from ``NUM_ROWS / LANES + pipeline_slack`` by the caller. + + Cycle counting + -------------- + ``query_ready`` is a combinational signal that goes low *immediately* after + the RisingEdge at which the query is accepted (the state machine + transitions from S_IDLE to S_SCAN). We therefore sample ``query_ready`` + **before** ``ReadOnly`` (i.e. at the RisingEdge time-point itself) to + capture the handshake. + + ``result_valid`` is a registered output that stays high until consumed, so + we sample it in the **settled phase** after ``ReadOnly``. + + ``latency_cycles`` is the number of RisingEdge events between the cycle + where the query is accepted and the cycle where ``result_valid`` is + observed. + """ + await wait_idle(dut) + + edge_count = 0 + dut.query_hash.value = int(query) + dut.query_valid.value = 1 + + # ── Phase 1: Accept handshake ─────────────────────────────────────── + # query_ready is combinational — sample before ReadOnly. + accept_edge = None + for _ in range(10): + await RisingEdge(dut.clk) + edge_count += 1 + q_ready = int(dut.query_ready.value) # sample before state transition + await ReadOnly() + await Timer(1, units="step") # exit ReadOnly for driving + if q_ready: + accept_edge = edge_count + break + assert accept_edge is not None, ( + "Query accept handshake timed out after 10 cycles" + ) + dut.query_valid.value = 0 + + # ── Phase 2: Wait for result_valid (measurement window) ───────────── + # result_valid is registered — sample after ReadOnly is fine. + result_edge = None + for _ in range(max_result_cycles): + await RisingEdge(dut.clk) + edge_count += 1 + await ReadOnly() + if int(dut.result_valid.value): + result_edge = edge_count + break + assert result_edge is not None, ( + f"Query result_valid timed out after {max_result_cycles} cycles " + f"(accepted at edge {accept_edge})" + ) + await Timer(1, units="step") + + latency_cycles = result_edge - accept_edge + + top1_index = int(dut.top1_index.value) + top1_score = int(dut.top1_score.value) + + # ── Phase 3: Consume result (pulse result_ready for one cycle) ────── + dut.result_ready.value = 1 + await RisingEdge(dut.clk) + edge_count += 1 + await ReadOnly() + await Timer(1, units="step") + dut.result_ready.value = 0 + + return top1_index, top1_score, latency_cycles, edge_count + + +def deterministic_rows(num_rows, hash_bits, query_hash): + """Create deterministic rows where only row 0 stores *query_hash*.""" + mask = (1 << hash_bits) - 1 + rows = [0] * num_rows + rows[0] = query_hash + for i in range(1, num_rows): + # Deterministic non-matching value; golden-ratio-like spread + val = ((i + 1) * 0x9E3779B97F4A7C15) & mask + if val == query_hash or val == 0: + val = ((val ^ query_hash) ^ 0xA5A5A5A5A5A5A5A5) & mask + if val == query_hash or val == 0: + val = (val ^ 1) & mask + rows[i] = val + return rows + + +# ── Performance Test ────────────────────────────────────────────────────────── + + +@cocotb.test() +async def cam_perf_benchmark(dut): + """Performance benchmark: measure query latency in cycles.""" + cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) + await reset_dut(dut) + + num_rows = dut_num_rows(dut) + hash_bits = dut_hash_bits(dut) + lanes = dut_lanes(dut) + write_noise_en = _get_param(dut, "WRITE_NOISE_EN", 1) + read_noise_en = _get_param(dut, "READ_NOISE_EN", 0) + + # ── Deterministic query ───────────────────────────────────────────── + query_hash = 0xAA55_AA55_AA55_AA55_AA55_AA55_AA55_AA55 + query_hash &= (1 << hash_bits) - 1 + if query_hash == 0: + query_hash = 1 + + rows = deterministic_rows(num_rows, hash_bits, query_hash) + await write_rows(dut, rows) + + # Bound: pipeline depth plus generous slack for read + drain stages + pipeline = _pipeline_depth(num_rows, lanes) + max_result_cycles = pipeline + 30 + + top1_index, top1_score, latency_cycles, total_cycles = ( + await query_once_with_latency(dut, query_hash, max_result_cycles) + ) + + # ── Performance assertions ────────────────────────────────────────── + assert latency_cycles > 0, ( + f"latency_cycles must be positive, got {latency_cycles}" + ) + assert total_cycles > 0, ( + f"total_cycles must be positive, got {total_cycles}" + ) + + # ── Correctness assertions (conditional on noise state) ───────────── + if not write_noise_en and not read_noise_en: + # Without noise: stored hash at row 0 == query_hash → exact match. + assert top1_index == 0, ( + f"Noise disabled: expected top1_index=0 (exact match), got " + f"{top1_index}" + ) + assert top1_score == hash_bits, ( + f"Noise disabled: expected top1_score={hash_bits}, got " + f"{top1_score}" + ) + else: + # With noise: write/read flip masks may corrupt stored values, so + # we cannot reliably assert the exact match. Instead, confirm a + # valid non-zero score was produced (the match engine ran). + assert top1_score > 0, ( + f"Noise enabled: expected top1_score > 0, got {top1_score}. " + "Match engine returned invalid result." + ) + dut._log.info( + "Noise enabled (WRITE_NOISE_EN=%s, READ_NOISE_EN=%s) — " + "skipping exact top1_index/top1_score assertion. " + "top1_index=%d top1_score=%d", + write_noise_en, read_noise_en, top1_index, top1_score, + ) + + # ── Machine-readable performance marker ───────────────────────────── + queries_per_cycle = 1.0 / total_cycles + dut._log.info( + "PERF_RESULT latency_cycles=%d total_cycles=%d " + "accepted_queries=1 completed_queries=1 " + "queries_per_cycle=%.6f status=pass", + latency_cycles, total_cycles, queries_per_cycle, + ) diff --git a/scripts/sweep_cam_perf.py b/scripts/sweep_cam_perf.py new file mode 100644 index 0000000..a6680fe --- /dev/null +++ b/scripts/sweep_cam_perf.py @@ -0,0 +1,762 @@ +#!/usr/bin/env python3 +"""CAM performance simulation sweep — data model, sweep matrix, and command builders.""" + +from __future__ import annotations + +import csv +import os +import re +import subprocess +import time +from dataclasses import dataclass +from datetime import date as date_type +from datetime import datetime +from pathlib import Path +from typing import Annotated + +import typer + +# Timeout for each external simulation invocation (30 minutes). +DEFAULT_COMMAND_TIMEOUT_SEC = 1800 + +# Cocotb test module that records PERF_RESULT metrics. +PERF_MODULE = "tests.test_cam_perf" + +# CSV column names for sweep result output — must match exactly. +CSV_FIELDNAMES = [ + "sweep_name", + "num_rows", + "hash_bits", + "lanes", + "latency_cycles", + "total_cycles", + "accepted_queries", + "completed_queries", + "queries_per_cycle", + "sim_time_sec", + "status", + "log_path", +] + + +# --------------------------------------------------------------------------- +# Dense sweep-matrix value sets for paper-quality figures. +NUM_ROWS_SWEEP_VALUES = (64, 128, 192, 256, 384, 512, 768, 1024) +HASH_BITS_SWEEP_VALUES = (64, 128, 192, 256, 320, 384, 448, 512) +LANES_SWEEP_VALUES = (2, 4, 8, 16, 32) + + +# --------------------------------------------------------------------------- +# Data model +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class SweepConfig: + """A single point in the CAM performance sweep matrix.""" + + sweep_name: str + num_rows: int + hash_bits: int + lanes: int + + +@dataclass(frozen=True) +class PerfRun: + """Result of executing a single sweep configuration.""" + + config: SweepConfig + status: str + returncode: int + duration_sec: float + log_path: Path + metrics: dict[str, str] + error: str = "" + + +# --------------------------------------------------------------------------- +# Default sweep matrix +# --------------------------------------------------------------------------- + + +def default_sweep_configs() -> list[SweepConfig]: + """Return the paper-supporting dense sweep matrix (19 unique configs). + + The matrix sweeps one dimension at a time: + * NUM_ROWS: ``(nr, 512, 16)`` for ``nr`` in ``NUM_ROWS_SWEEP_VALUES`` + * HASH_BITS: ``(512, hb, 16)`` for ``hb`` in ``HASH_BITS_SWEEP_VALUES`` + * LANES: ``(512, 512, la)`` for ``la`` in ``LANES_SWEEP_VALUES`` + + Overlapping triples are deduplicated preserving first occurrence. + """ + candidates = ( + [SweepConfig("num_rows", nr, 512, 16) for nr in NUM_ROWS_SWEEP_VALUES] + + [SweepConfig("hash_bits", 512, hb, 16) for hb in HASH_BITS_SWEEP_VALUES] + + [SweepConfig("lanes", 512, 512, la) for la in LANES_SWEEP_VALUES] + ) + seen: set[tuple[int, int, int]] = set() + result: list[SweepConfig] = [] + for cfg in candidates: + key = (cfg.num_rows, cfg.hash_bits, cfg.lanes) + if key not in seen: + seen.add(key) + result.append(cfg) + return result + + +# --------------------------------------------------------------------------- +# Sweep membership helper +# --------------------------------------------------------------------------- + + +def logical_sweeps_for_config(config: SweepConfig) -> tuple[str, ...]: + """Return the logical sweep-group memberships for *config*. + + A single config may participate in multiple sweep dimensions. This is + especially important for the baseline config ``(512, 512, 16)`` whose + singleton CSV row has ``sweep_name='num_rows'`` (first occurrence wins + during dedup), yet logically belongs to **all three** sweep groups. + + The predicates reference the module-level dense-value constants: + * ``'num_rows'`` – ``HASH_BITS=512, LANES=16, NUM_ROWS ∈ NUM_ROWS_SWEEP_VALUES`` + * ``'hash_bits'`` – ``NUM_ROWS=512, LANES=16, HASH_BITS ∈ HASH_BITS_SWEEP_VALUES`` + * ``'lanes'`` – ``NUM_ROWS=512, HASH_BITS=512, LANES ∈ LANES_SWEEP_VALUES`` + + Downstream plot / analysis code **must** use this helper to derive sweep + groups instead of filtering solely by ``SweepConfig.sweep_name``; otherwise + the baseline ``(512,512,16)`` would be missing from ``hash_bits`` and + ``lanes`` group queries. + """ + memberships: list[str] = [] + nr, hb, la = config.num_rows, config.hash_bits, config.lanes + + if hb == 512 and la == 16 and nr in NUM_ROWS_SWEEP_VALUES: + memberships.append("num_rows") + if nr == 512 and la == 16 and hb in HASH_BITS_SWEEP_VALUES: + memberships.append("hash_bits") + if nr == 512 and hb == 512 and la in LANES_SWEEP_VALUES: + memberships.append("lanes") + + return tuple(memberships) + + +# --------------------------------------------------------------------------- +# Log parsing +# --------------------------------------------------------------------------- + + +_PERF_EXPECTED_KEYS = frozenset({ + "latency_cycles", + "total_cycles", + "accepted_queries", + "completed_queries", + "queries_per_cycle", + "status", +}) + + +def parse_perf_result(log_text: str) -> dict[str, str]: + """Parse the first *complete* ``PERF_RESULT`` marker line from a simulation log. + + Scans all lines for a ``PERF_RESULT`` marker (standalone word). For each + such line, extracts ``key=value`` tokens and checks whether all expected + keys (``latency_cycles``, ``total_cycles``, ``accepted_queries``, + ``completed_queries``, ``queries_per_cycle``, ``status``) are present. + Returns the dict from the **first** line with a complete set, or ``{}`` + if no complete marker exists in the entire log. + """ + _marker_re = re.compile(r"\bPERF_RESULT\b") + for line in log_text.splitlines(): + if _marker_re.search(line): + tokens = line.split() + parsed: dict[str, str] = {} + for token in tokens: + if "=" in token: + key, _, value = token.partition("=") + parsed[key] = value + if _PERF_EXPECTED_KEYS.issubset(parsed.keys()): + return parsed + return {} + + +# --------------------------------------------------------------------------- +# Command builders +# --------------------------------------------------------------------------- + + +def build_make_command(config: SweepConfig) -> list[str]: + """Build a ``make`` command to run the perf test module with *config*.""" + return [ + "make", + "-C", + "hw/sim", + f"NUM_ROWS={config.num_rows}", + f"HASH_BITS={config.hash_bits}", + f"LANES={config.lanes}", + f"COCOTB_TEST_MODULES={PERF_MODULE}", + ] + + +# --------------------------------------------------------------------------- +# Execution helpers +# --------------------------------------------------------------------------- + + +def _run_subprocess( + cmd: list[str], + *, + cwd: Path, + env: dict[str, str], + timeout: int, +) -> tuple[subprocess.CompletedProcess[str] | None, float, str | None]: + """Execute *cmd*, capture output, enforce *timeout*. + + Returns ``(result, duration_sec, error_msg)``. + On success *result* is a ``CompletedProcess`` and *error_msg* is ``None``. + On failure (timeout / OSError / SubprocessError) *result* is ``None`` + and *error_msg* describes what happened. + """ + start = time.monotonic() + try: + result = subprocess.run( + cmd, + cwd=cwd, + env=env, + text=True, + capture_output=True, + timeout=timeout, + ) + except subprocess.TimeoutExpired as exc: + end = time.monotonic() + partial = "" + if exc.stdout: + partial += f"\n# [partial stdout before timeout]\n{exc.stdout}" + if exc.stderr: + partial += f"\n# [partial stderr before timeout]\n{exc.stderr}" + return None, round(end - start, 3), f"timed out after {timeout}s{partial}" + except (OSError, subprocess.SubprocessError) as exc: + end = time.monotonic() + return None, round(end - start, 3), str(exc) + end = time.monotonic() + return result, round(end - start, 3), None + + +def run_one_config( + config: SweepConfig, + *, + output_dir: Path, + repo_root: Path, + timeout: int = DEFAULT_COMMAND_TIMEOUT_SEC, +) -> PerfRun: + """Execute a single sweep configuration and return the result. + + Steps: + 1. Create logs directory. + 2. Validate ``NUM_ROWS % LANES == 0``; return fail immediately if invalid. + 3. Run ``make -C hw/sim clean``. + 4. Build and run the simulation command via ``build_make_command(config)``. + 5. Capture combined stdout/stderr into a log file named + ``_r_b_l.log``. + 6. Parse ``PERF_RESULT`` from the combined log. + 7. Determine pass/fail: pass only if returncode == 0 and metrics status == 'pass'. + """ + logs_dir = output_dir / "logs" + logs_dir.mkdir(parents=True, exist_ok=True) + + log_filename = ( + f"{config.sweep_name}_{config.num_rows}r_{config.hash_bits}b_{config.lanes}l.log" + ) + log_path = logs_dir / log_filename + relative_log = Path("logs") / log_filename + + # ── Validation: NUM_ROWS must be divisible by LANES ────────────────── + if config.num_rows % config.lanes != 0: + log_path.write_text( + f"# ERROR: NUM_ROWS ({config.num_rows}) not divisible by LANES ({config.lanes})\n" + ) + return PerfRun( + config=config, + status="fail", + returncode=-1, + duration_sec=0.0, + log_path=relative_log, + metrics={}, + error=f"NUM_ROWS ({config.num_rows}) not divisible by LANES ({config.lanes})", + ) + + env = os.environ.copy() + log_chunks: list[str] = [] + + # ── Clean step ─────────────────────────────────────────────────────── + clean_cmd = ["make", "-C", "hw/sim", "clean"] + log_chunks.append(f"# Command: {' '.join(clean_cmd)}\n") + clean_result, clean_duration, clean_err = _run_subprocess( + clean_cmd, cwd=repo_root, env=env, timeout=timeout, + ) + if clean_result is not None: + log_chunks.append(clean_result.stdout) + log_chunks.append(clean_result.stderr) + else: + log_chunks.append(f"# ERROR: {clean_err}\n") + + if clean_err is not None or ( + clean_result is not None and clean_result.returncode != 0 + ): + returncode = clean_result.returncode if clean_result is not None else -1 + log_path.write_text("".join(log_chunks)) + return PerfRun( + config=config, + status="fail", + returncode=returncode, + duration_sec=clean_duration, + log_path=relative_log, + metrics={}, + error=clean_err or f"make clean failed with returncode {returncode}", + ) + + # ── Main simulation command ────────────────────────────────────────── + command = build_make_command(config) + log_chunks.append(f"# Command: {' '.join(command)}\n") + main_result, duration, main_err = _run_subprocess( + command, cwd=repo_root, env=env, timeout=timeout, + ) + + if main_result is not None: + log_chunks.append(main_result.stdout) + log_chunks.append(main_result.stderr) + else: + log_chunks.append(f"# ERROR: {main_err}\n") + + combined_log = "".join(log_chunks) + log_path.write_text(combined_log) + + # ── Parse PERF_RESULT ──────────────────────────────────────────────── + metrics = parse_perf_result(combined_log) + + if main_err is not None: + status = "fail" + returncode = -1 + error = main_err + else: + returncode = main_result.returncode + metrics_status = metrics.get("status", "") + if returncode == 0 and metrics_status == "pass": + status = "pass" + error = "" + else: + status = "fail" + if returncode != 0: + error = f"simulation returned non-zero exit code {returncode}" + elif metrics_status != "pass": + error = f"PERF_RESULT status is '{metrics_status}'" + else: + error = "unknown failure" + + return PerfRun( + config=config, + status=status, + returncode=returncode, + duration_sec=duration, + log_path=relative_log, + metrics=metrics, + error=error, + ) + + +# --------------------------------------------------------------------------- +# CSV helpers +# --------------------------------------------------------------------------- + + +def run_to_row(run: PerfRun) -> dict[str, str]: + """Convert a ``PerfRun`` to a CSV row dict matching ``CSV_FIELDNAMES``.""" + return { + "sweep_name": run.config.sweep_name, + "num_rows": str(run.config.num_rows), + "hash_bits": str(run.config.hash_bits), + "lanes": str(run.config.lanes), + "latency_cycles": run.metrics.get("latency_cycles", ""), + "total_cycles": run.metrics.get("total_cycles", ""), + "accepted_queries": run.metrics.get("accepted_queries", ""), + "completed_queries": run.metrics.get("completed_queries", ""), + "queries_per_cycle": run.metrics.get("queries_per_cycle", ""), + "sim_time_sec": str(run.duration_sec), + "status": run.status, + "log_path": str(run.log_path), + } + + +def write_csv(output_dir: Path, runs: list[PerfRun]) -> None: + """Write ``sweep.csv`` with ``CSV_FIELDNAMES`` header and one row per run.""" + path = output_dir / "sweep.csv" + with open(path, "w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=CSV_FIELDNAMES) + writer.writeheader() + for run in runs: + writer.writerow(run_to_row(run)) + + +# --------------------------------------------------------------------------- +# Output helpers +# --------------------------------------------------------------------------- + + +def _float_or_none(value: str) -> float | None: + """Parse *value* as ``float``, returning ``None`` on failure.""" + try: + return float(value) + except (ValueError, TypeError): + return None + + +# --------------------------------------------------------------------------- +# Nature-like plot style constants +# --------------------------------------------------------------------------- + +PLOT_NAVY = "#243B53" +PLOT_SLATE = "#7A8793" +PLOT_TEAL = "#4FA7A3" +PLOT_GRID = "#D7DEE5" +PLOT_TEXT = "#1F2933" + + +def _style_perf_axis(ax, *, title: str, xlabel: str, ylabel: str) -> None: + """Apply Nature-like styling to a performance figure axis.""" + ax.set_title(title, fontsize=10, color=PLOT_TEXT, pad=8) + ax.set_xlabel(xlabel, fontsize=9, color=PLOT_TEXT) + ax.set_ylabel(ylabel, fontsize=9, color=PLOT_TEXT) + ax.grid(True, color=PLOT_GRID, linewidth=0.6, alpha=0.8) + ax.spines["top"].set_visible(False) + ax.spines["right"].set_visible(False) + ax.spines["left"].set_color(PLOT_SLATE) + ax.spines["bottom"].set_color(PLOT_SLATE) + ax.tick_params(axis="both", colors=PLOT_TEXT, labelsize=8, width=0.6) + + +def _plot_points( + runs: list[PerfRun], group: str, x_getter, metric: str, +) -> list[tuple[int, float]]: + """Filter *runs* to pass runs in *group* and return sorted ``(x, y)`` points.""" + points: list[tuple[int, float]] = [] + for run in runs: + if run.status != "pass" or group not in logical_sweeps_for_config(run.config): + continue + value = _float_or_none(run.metrics.get(metric, "")) + if value is None: + continue + points.append((int(x_getter(run.config)), value)) + return sorted(points, key=lambda item: item[0]) + + +def write_summary(output_dir: Path, runs: list[PerfRun]) -> None: + """Write ``sweep_summary.md`` in Chinese with summary table and disclaimer.""" + total = len(runs) + passed = sum(1 for r in runs if r.status == "pass") + failed = total - passed + failed_runs = [r for r in runs if r.status == "fail"] + + lines: list[str] = [] + lines.append("# CAM 性能扫描摘要") + lines.append("") + lines.append(f"**输出目录:** {output_dir}") + lines.append(f"**总运行数:** {total}") + lines.append(f"**通过数:** {passed}") + lines.append(f"**失败数:** {failed}") + lines.append("") + lines.append("## 性能结果") + lines.append("") + lines.append("| 扫描分组 | NUM_ROWS | HASH_BITS | LANES | 延迟(周期数) | 查询量/周期 | 状态 | 日志 |") + lines.append("|----------|----------|-----------|-------|-------------|-------------|------|------|") + for r in runs: + lines.append( + f"| {r.config.sweep_name} | {r.config.num_rows} | {r.config.hash_bits} " + f"| {r.config.lanes} | {r.metrics.get('latency_cycles', '')} " + f"| {r.metrics.get('queries_per_cycle', '')} | {r.status} " + f"| {r.log_path} |" + ) + lines.append("") + lines.append("## 失败列表") + lines.append("") + if failed_runs: + for r in failed_runs: + err = r.error or "无" + lines.append(f"- **{r.config.sweep_name}** (`{r.log_path}`): {err}") + else: + lines.append("- 无") + lines.append("") + lines.append("## 免责声明") + lines.append("") + lines.append( + "本报告为 **Verilator/Cocotb 仿真** 结果,非 FPGA 板级实测性能、" + "资源利用率或 Fmax。所有数据仅供架构评估参考,不代表最终 FPGA 实现指标。" + ) + lines.append("") + + path = output_dir / "sweep_summary.md" + path.write_text("\n".join(lines)) + + +def write_notes(output_dir: Path, runs: list[PerfRun]) -> None: + """Write ``ch6_3_cam_perf_notes.md`` in Chinese with trend analysis.""" + lines: list[str] = [] + lines.append("# 第 6.3 节 CAM 性能仿真笔记") + lines.append("") + lines.append("## 说明") + lines.append("") + lines.append( + "以下结果基于 **Verilator/Cocotb 仿真** 环境,旨在评估 CAM 设计的查询延迟" + "(latency_cycles)和吞吐量(queries_per_cycle)趋势。所有数据均为仿真" + "结果,**非 FPGA 板级实测性能**。本文档不提供 LUT、FF、BRAM 或 Fmax 等" + "资源利用率或时序结论。" + ) + lines.append("") + lines.append("## 参数趋势分析") + lines.append("") + lines.append("### NUM_ROWS(CAM 行数)") + lines.append("") + nr_runs = [ + r for r in runs + if "num_rows" in logical_sweeps_for_config(r.config) and r.status == "pass" + ] + if nr_runs: + for r in sorted(nr_runs, key=lambda x: x.config.num_rows): + lat = r.metrics.get("latency_cycles", "N/A") + qpc = r.metrics.get("queries_per_cycle", "N/A") + lines.append(f"- NUM_ROWS={r.config.num_rows}: 延迟={lat} 周期, 吞吐量={qpc} 查询/周期") + else: + lines.append("- 无通过数据") + lines.append("") + lines.append("### HASH_BITS(哈希位宽)") + lines.append("") + hb_runs = [ + r for r in runs + if "hash_bits" in logical_sweeps_for_config(r.config) and r.status == "pass" + ] + if hb_runs: + for r in sorted(hb_runs, key=lambda x: x.config.hash_bits): + lat = r.metrics.get("latency_cycles", "N/A") + qpc = r.metrics.get("queries_per_cycle", "N/A") + lines.append(f"- HASH_BITS={r.config.hash_bits}: 延迟={lat} 周期, 吞吐量={qpc} 查询/周期") + else: + lines.append("- 无通过数据") + lines.append("") + lines.append("### LANES(流水线通路数)") + lines.append("") + la_runs = [ + r for r in runs + if "lanes" in logical_sweeps_for_config(r.config) and r.status == "pass" + ] + if la_runs: + for r in sorted(la_runs, key=lambda x: x.config.lanes): + lat = r.metrics.get("latency_cycles", "N/A") + qpc = r.metrics.get("queries_per_cycle", "N/A") + lines.append(f"- LANES={r.config.lanes}: 延迟={lat} 周期, 吞吐量={qpc} 查询/周期") + else: + lines.append("- 无通过数据") + lines.append("") + lines.append("## 免责声明") + lines.append("") + lines.append( + "本文档中的数据来源于 Verilator/Cocotb 仿真,用于观察性能趋势。" + "实际 FPGA 实现的性能、资源利用率和最大工作频率(Fmax)可能因综合选项、" + "布局布线、芯片型号等因素而显著不同。请勿将本文档中的数据直接用作 FPGA " + "实现指标。" + ) + lines.append("") + + path = output_dir / "ch6_3_cam_perf_notes.md" + path.write_text("\n".join(lines)) + + +def plot_figures(output_dir: Path, runs: list[PerfRun]) -> None: + """Generate performance trend figures under ``figures/``. + + Three single-panel PNGs and one 1×3 multi-panel PNG are produced + when at least 2 pass rows exist in a logical sweep group: + - ``cam_num_rows_vs_latency.png`` + - ``cam_hash_bits_vs_latency.png`` + - ``cam_lanes_vs_throughput.png`` + - ``cam_perf_multipanel.png`` + + Uses ``logical_sweeps_for_config`` so the baseline ``(512,512,16)`` + appears in all three groups. + """ + pass_runs = [r for r in runs if r.status == "pass"] + if not pass_runs: + return + + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + + # White / Nature-like rcParams defaults. + plt.rcParams.update({ + "figure.facecolor": "white", + "axes.facecolor": "white", + "savefig.facecolor": "white", + "font.family": "DejaVu Sans", + "axes.linewidth": 0.8, + }) + + fig_dir = output_dir / "figures" + fig_dir.mkdir(parents=True, exist_ok=True) + + # ── Define the three logical groups ────────────────────────────────── + groups: list[dict] = [ + { + "group": "num_rows", + "x_getter": lambda c: c.num_rows, + "metric": "latency_cycles", + "xlabel": "NUM_ROWS", + "ylabel": "Latency (cycles)", + "title": "CAM Query Latency vs NUM_ROWS\n(HASH_BITS=512, LANES=16)", + "filename": "cam_num_rows_vs_latency.png", + "marker": "o", + }, + { + "group": "hash_bits", + "x_getter": lambda c: c.hash_bits, + "metric": "latency_cycles", + "xlabel": "HASH_BITS", + "ylabel": "Latency (cycles)", + "title": "CAM Query Latency vs HASH_BITS\n(NUM_ROWS=512, LANES=16)", + "filename": "cam_hash_bits_vs_latency.png", + "marker": "s", + }, + { + "group": "lanes", + "x_getter": lambda c: c.lanes, + "metric": "queries_per_cycle", + "xlabel": "LANES", + "ylabel": "Queries / Cycle", + "title": "CAM Throughput vs LANES\n(NUM_ROWS=512, HASH_BITS=512)", + "filename": "cam_lanes_vs_throughput.png", + "marker": "^", + }, + ] + + # ── Individual figures ─────────────────────────────────────────────── + for g in groups: + points = _plot_points(pass_runs, g["group"], g["x_getter"], g["metric"]) + if len(points) < 2: + continue + xs, ys = zip(*points) + fig, ax = plt.subplots(figsize=(4.2, 3.0)) + ax.plot( + xs, ys, + color=PLOT_NAVY, linewidth=1.6, + marker=g["marker"], markersize=4.5, + markerfacecolor=PLOT_TEAL, markeredgecolor="white", markeredgewidth=0.7, + ) + _style_perf_axis(ax, title=g["title"], xlabel=g["xlabel"], ylabel=g["ylabel"]) + fig.tight_layout(pad=0.7) + fig.savefig(fig_dir / g["filename"], dpi=300, bbox_inches="tight") + plt.close(fig) + + # ── Multi-panel figure (1 × 3) ─────────────────────────────────────── + fig, axes = plt.subplots(1, 3, figsize=(10.8, 3.0), constrained_layout=True) + for ax, g in zip(axes, groups): + points = _plot_points(pass_runs, g["group"], g["x_getter"], g["metric"]) + if len(points) < 2: + ax.text(0.5, 0.5, "insufficient data", ha="center", va="center", + transform=ax.transAxes, fontsize=8, color=PLOT_SLATE) + continue + xs, ys = zip(*points) + ax.plot( + xs, ys, + color=PLOT_NAVY, linewidth=1.6, + marker=g["marker"], markersize=4.5, + markerfacecolor=PLOT_TEAL, markeredgecolor="white", markeredgewidth=0.7, + ) + _style_perf_axis(ax, title=g["title"], xlabel=g["xlabel"], ylabel=g["ylabel"]) + fig.savefig(fig_dir / "cam_perf_multipanel.png", dpi=300, bbox_inches="tight") + plt.close(fig) + + +# --------------------------------------------------------------------------- +# Orchestrator +# --------------------------------------------------------------------------- + + +def run_all( + configs: list[SweepConfig], + output_root: Path = Path("outputs/cam_perf"), + date: str | None = None, + repo_root: Path | None = None, +) -> int: + """Run all sweep configurations sequentially, write CSV, return 0 if all pass. + + Output is written under ``*output_root* / *date*``. + """ + if repo_root is None: + repo_root = Path(__file__).resolve().parent.parent + output_root = Path(output_root) + if not output_root.is_absolute(): + output_root = repo_root / output_root + if date is None: + date = datetime.now().strftime("%Y-%m-%d") + output_dir = output_root / date + output_dir.mkdir(parents=True, exist_ok=True) + + runs: list[PerfRun] = [] + for config in configs: + run = run_one_config(config, output_dir=output_dir, repo_root=repo_root) + runs.append(run) + + write_csv(output_dir, runs) + write_summary(output_dir, runs) + write_notes(output_dir, runs) + plot_figures(output_dir, runs) + + all_pass = all(run.status == "pass" for run in runs) + return 0 if all_pass else 1 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def script_repo_root() -> Path: + """Return the repository root (two levels up from ``scripts/sweep_cam_perf.py``).""" + return Path(__file__).resolve().parents[1] + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +app = typer.Typer( + add_completion=False, + help=( + "Run CAM performance simulation sweep " + "and collect paper-ready outputs." + ), +) + + +@app.command() +def main( + date: Annotated[str | None, typer.Option("--date", help="Output date directory. Defaults to today.")] = None, + single: Annotated[bool, typer.Option("--single", help="Run a single config instead of the full sweep.")] = False, + num_rows: Annotated[int, typer.Option("--num-rows", help="CAM row count.", min=1)] = 512, + hash_bits: Annotated[int, typer.Option("--hash-bits", help="Hash width.", min=1)] = 512, + lanes: Annotated[int, typer.Option("--lanes", help="CAM lane count.", min=1)] = 4, +) -> None: + """Run the CAM performance simulation sweep.""" + if single: + configs = [SweepConfig("single", num_rows, hash_bits, lanes)] + else: + configs = default_sweep_configs() + + run_date = date or date_type.today().isoformat() + exit_code = run_all( + configs=configs, + date=run_date, + repo_root=script_repo_root(), + ) + raise typer.Exit(exit_code) + + +if __name__ == "__main__": + app() diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..1a703f7 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,21 @@ +"""pytest configuration — ensure ``scripts/`` is importable via package syntax. + +The ``scripts`` directory is a PEP 420 implicit namespace package (no +``__init__.py``). Because ``uv run pytest`` does **not** add the project root +to ``sys.path``, ``from scripts.sweep_cam_perf import ...`` would otherwise +fail with ``ModuleNotFoundError``. + +This conftest inserts the repository root at the front of ``sys.path`` so that +``scripts`` is discoverable as a top-level package. +""" + +from __future__ import annotations + +import sys +from pathlib import Path + +_repo_root = Path(__file__).resolve().parent.parent +_repo_root_str = str(_repo_root) + +if _repo_root_str not in sys.path: + sys.path.insert(0, _repo_root_str) diff --git a/tests/test_sweep_cam_perf.py b/tests/test_sweep_cam_perf.py new file mode 100644 index 0000000..a4da2dd --- /dev/null +++ b/tests/test_sweep_cam_perf.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +"""Fast unit tests for scripts/sweep_cam_perf.py (sweep helpers).""" + +from __future__ import annotations + +import subprocess +from pathlib import Path + +from scripts.sweep_cam_perf import ( + CSV_FIELDNAMES, + PerfRun, + SweepConfig, + _run_subprocess, + build_make_command, + default_sweep_configs, + logical_sweeps_for_config, + parse_perf_result, + plot_figures, + run_all, +) + + +# --------------------------------------------------------------------------- +# parse_perf_result +# --------------------------------------------------------------------------- + + +def test_parse_perf_result_extracts_metrics(): + log = ( + "PERF_RESULT latency_cycles=148 total_cycles=158 " + "accepted_queries=1 completed_queries=1 " + "queries_per_cycle=0.006329 status=pass" + ) + result = parse_perf_result(log) + assert result == { + "latency_cycles": "148", + "total_cycles": "158", + "accepted_queries": "1", + "completed_queries": "1", + "queries_per_cycle": "0.006329", + "status": "pass", + } + + +def test_parse_perf_result_returns_empty_when_missing(): + log = "some random simulation output without the marker" + assert parse_perf_result(log) == {} + + +# --------------------------------------------------------------------------- +# default_sweep_configs +# --------------------------------------------------------------------------- + + +def test_default_sweep_configs_are_paper_supporting_dense_matrix(): + configs = default_sweep_configs() + assert len(configs) == 19 + + tuples = {(c.num_rows, c.hash_bits, c.lanes) for c in configs} + + # NUM_ROWS sweep (lanes fixed at 16) + for nr in (64, 128, 192, 256, 384, 512, 768, 1024): + assert (nr, 512, 16) in tuples, f"Missing NUM_ROWS config {(nr, 512, 16)}" + + # HASH_BITS sweep (num_rows fixed at 512, lanes fixed at 16) + for hb in (64, 128, 192, 256, 320, 384, 448, 512): + assert (512, hb, 16) in tuples, f"Missing HASH_BITS config {(512, hb, 16)}" + + # LANES sweep (num_rows and hash_bits fixed at 512) + for la in (2, 4, 8, 16, 32): + assert (512, 512, la) in tuples, f"Missing LANES config {(512, 512, la)}" + + assert len(tuples) == 19 + # All configs must satisfy num_rows % lanes == 0 + for c in configs: + assert c.num_rows % c.lanes == 0, ( + f"num_rows={c.num_rows} not divisible by lanes={c.lanes} for {c}" + ) + + +# --------------------------------------------------------------------------- +# logical_sweeps_for_config +# --------------------------------------------------------------------------- + + +def test_logical_sweeps_baseline_belongs_to_all_three_groups(): + """The baseline (512,512,16) config belongs to all three sweep groups, + even though its CSV sweep_name is 'num_rows' (first-occurrence dedup).""" + configs = default_sweep_configs() + # Locate the shared baseline config. + baseline = next(c for c in configs if (c.num_rows, c.hash_bits, c.lanes) == (512, 512, 16)) + groups = logical_sweeps_for_config(baseline) + assert set(groups) == {"num_rows", "hash_bits", "lanes"}, ( + f"Baseline should belong to all three groups, got {groups}" + ) + assert len(configs) == 19, "Sweep config count must remain 19" + + +# --------------------------------------------------------------------------- +# build_make_command +# --------------------------------------------------------------------------- + + +def test_build_make_command_uses_perf_test_module(): + cfg = SweepConfig("smoke", 128, 128, 4) + cmd = build_make_command(cfg) + assert cmd == [ + "make", + "-C", + "hw/sim", + "NUM_ROWS=128", + "HASH_BITS=128", + "LANES=4", + "COCOTB_TEST_MODULES=tests.test_cam_perf", + ] + + +# --------------------------------------------------------------------------- +# CSV_FIELDNAMES +# --------------------------------------------------------------------------- + + +def test_csv_fieldnames_match_spec(): + assert CSV_FIELDNAMES == [ + "sweep_name", + "num_rows", + "hash_bits", + "lanes", + "latency_cycles", + "total_cycles", + "accepted_queries", + "completed_queries", + "queries_per_cycle", + "sim_time_sec", + "status", + "log_path", + ] + + +# --------------------------------------------------------------------------- +# _run_subprocess — timeout preserves partial output +# --------------------------------------------------------------------------- + + +def test_run_subprocess_timeout_preserves_partial_output(monkeypatch): + """Timeout error string includes partial stdout/stderr from the process.""" + + def mock_run(*args, **kwargs): + raise subprocess.TimeoutExpired( + cmd=["mock"], + timeout=1, + output="partial stdout data", + stderr="partial stderr data", + ) + + monkeypatch.setattr(subprocess, "run", mock_run) + + result, duration, error = _run_subprocess( + ["mock"], cwd=Path("/tmp"), env={}, timeout=1, + ) + + assert result is None + assert duration >= 0 + assert "timed out after 1s" in error + assert "[partial stdout before timeout]" in error + assert "partial stdout data" in error + assert "[partial stderr before timeout]" in error + assert "partial stderr data" in error + + +def test_run_subprocess_timeout_without_partial_output(monkeypatch): + """Timeout error is clean when the subprocess produced no output.""" + + def mock_run(*args, **kwargs): + raise subprocess.TimeoutExpired(cmd=["mock"], timeout=5) + + monkeypatch.setattr(subprocess, "run", mock_run) + + result, duration, error = _run_subprocess( + ["mock"], cwd=Path("/tmp"), env={}, timeout=5, + ) + + assert result is None + assert "timed out after 5s" in error + assert "[partial stdout" not in error + + +# --------------------------------------------------------------------------- +# run_all — output_root anchoring +# --------------------------------------------------------------------------- + + +def _make_mock_run_one_config(status="pass"): + """Return a ``run_one_config`` stand-in that never invokes a subprocess.""" + + def mock(config, *, output_dir, repo_root, timeout=1800): + return PerfRun( + config=config, + status=status, + returncode=0 if status == "pass" else 1, + duration_sec=0.0, + log_path=Path("logs/test.log"), + metrics={"status": status}, + ) + + return mock + + +def test_run_all_anchors_relative_output_root_to_repo_root(tmp_path, monkeypatch): + """A relative ``output_root`` is resolved under ``repo_root``.""" + monkeypatch.setattr( + "scripts.sweep_cam_perf.run_one_config", + _make_mock_run_one_config("pass"), + ) + + configs = [SweepConfig("test", 512, 512, 16)] + exit_code = run_all( + configs, + output_root="outputs/cam_perf", + date="2026-06-01", + repo_root=tmp_path, + ) + + expected_csv = tmp_path / "outputs" / "cam_perf" / "2026-06-01" / "sweep.csv" + assert expected_csv.exists(), f"sweep.csv not found at {expected_csv}" + assert exit_code == 0 + + +def test_run_all_uses_absolute_output_root_unmodified(tmp_path, monkeypatch): + """An absolute ``output_root`` is used as-is, not anchored under repo_root.""" + monkeypatch.setattr( + "scripts.sweep_cam_perf.run_one_config", + _make_mock_run_one_config("pass"), + ) + + configs = [SweepConfig("test", 512, 512, 16)] + abs_output = tmp_path / "custom_dir" + repo_root = Path("/nonexistent/repo") + + exit_code = run_all( + configs, + output_root=abs_output, + date="2026-06-01", + repo_root=repo_root, + ) + + expected_csv = abs_output / "2026-06-01" / "sweep.csv" + assert expected_csv.exists(), f"sweep.csv not found at {expected_csv}" + assert exit_code == 0 + + +# --------------------------------------------------------------------------- +# plot_figures — required PNG outputs +# --------------------------------------------------------------------------- + + +def test_plot_figures_creates_required_pngs(tmp_path): + """plot_figures creates four PNGs under figures/ when enough pass runs exist + across all three logical sweep groups. No exception with Agg backend.""" + configs = [ + SweepConfig("num_rows", 128, 512, 16), + SweepConfig("num_rows", 256, 512, 16), + SweepConfig("num_rows", 512, 512, 16), # baseline: all three groups + SweepConfig("hash_bits", 512, 128, 16), + SweepConfig("hash_bits", 512, 256, 16), + SweepConfig("lanes", 512, 512, 4), + SweepConfig("lanes", 512, 512, 8), + ] + runs = [ + PerfRun( + config=cfg, + status="pass", + returncode=0, + duration_sec=1.0, + log_path=Path("logs/test.log"), + metrics={ + "latency_cycles": str(50 + cfg.num_rows % 100), + "total_cycles": str(100 + cfg.num_rows % 100), + "accepted_queries": "10", + "completed_queries": "10", + "queries_per_cycle": "0.05", + "status": "pass", + }, + ) + for cfg in configs + ] + + plot_figures(tmp_path, runs) + + fig_dir = tmp_path / "figures" + assert fig_dir.is_dir() + assert (fig_dir / "cam_num_rows_vs_latency.png").exists() + assert (fig_dir / "cam_hash_bits_vs_latency.png").exists() + assert (fig_dir / "cam_lanes_vs_throughput.png").exists() + assert (fig_dir / "cam_perf_multipanel.png").exists()