mirror of
https://github.com/SikongJueluo/Mini-Nav.git
synced 2026-07-12 20:15:31 +08:00
feat(sim): add CAM performance sweep test infrastructure
- hw/sim/tests/test_cam_perf.py: new cocotb perf test with bounded wait helpers - scripts/sweep_cam_perf.py: sweep data model, matrix, and make command builders - tests/test_sweep_cam_perf.py: unit tests for sweep helpers - tests/conftest.py: pytest path configuration for scripts package - hw/sim/Makefile: deterministic noise params override for perf test compatibility
This commit is contained in:
@@ -27,6 +27,18 @@ READ_NOISE_RATE_NUM ?=
|
|||||||
READ_NOISE_RATE_DEN ?=
|
READ_NOISE_RATE_DEN ?=
|
||||||
READ_NOISE_BITS ?=
|
READ_NOISE_BITS ?=
|
||||||
|
|
||||||
|
# Perf-test compatibility override: when running cam_perf, force deterministic
|
||||||
|
# noise parameters so noise_mask_grouped.sv elaborates correctly for perf
|
||||||
|
# sweep HASH_BITS values (64, 128, 192, 256, 320, 384, 448, 512 — all
|
||||||
|
# divisible by 64 such that HASH_BITS / NOISE_BITS == 64 in the RTL).
|
||||||
|
# This is a testbench parameter, not a hardware performance/resource claim.
|
||||||
|
ifeq ($(COCOTB_TEST_MODULES),tests.test_cam_perf)
|
||||||
|
WRITE_NOISE_EN := 0
|
||||||
|
READ_NOISE_EN := 0
|
||||||
|
WRITE_NOISE_BITS := $(shell echo $$(( $(HASH_BITS) / 64 )))
|
||||||
|
READ_NOISE_BITS := $(shell echo $$(( $(HASH_BITS) / 64 )))
|
||||||
|
endif
|
||||||
|
|
||||||
ifneq ($(strip $(WRITE_NOISE_EN)),)
|
ifneq ($(strip $(WRITE_NOISE_EN)),)
|
||||||
COMPILE_ARGS += -GWRITE_NOISE_EN=$(WRITE_NOISE_EN)
|
COMPILE_ARGS += -GWRITE_NOISE_EN=$(WRITE_NOISE_EN)
|
||||||
endif
|
endif
|
||||||
|
|||||||
306
hw/sim/tests/test_cam_perf.py
Normal file
306
hw/sim/tests/test_cam_perf.py
Normal file
@@ -0,0 +1,306 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import cocotb
|
||||||
|
from cocotb.clock import Clock
|
||||||
|
from cocotb.triggers import RisingEdge, ReadOnly, Timer
|
||||||
|
|
||||||
|
|
||||||
|
# ── Helpers (local reimplementation — no imports from test_cam_basic) ─────────
|
||||||
|
|
||||||
|
|
||||||
|
def _get_param(dut, name, default=None):
|
||||||
|
"""Read a Verilator-exposed parameter from the DUT."""
|
||||||
|
try:
|
||||||
|
val = getattr(dut, name, None)
|
||||||
|
if val is not None:
|
||||||
|
return int(val.value)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
return default
|
||||||
|
|
||||||
|
|
||||||
|
def dut_num_rows(dut):
|
||||||
|
val = _get_param(dut, "NUM_ROWS", None)
|
||||||
|
if val is not None:
|
||||||
|
return val
|
||||||
|
return 1 << len(dut.wr_addr)
|
||||||
|
|
||||||
|
|
||||||
|
def dut_hash_bits(dut):
|
||||||
|
val = _get_param(dut, "HASH_BITS", None)
|
||||||
|
if val is not None:
|
||||||
|
return val
|
||||||
|
return len(dut.write_hash)
|
||||||
|
|
||||||
|
|
||||||
|
def dut_lanes(dut):
|
||||||
|
val = _get_param(dut, "LANES", None)
|
||||||
|
if val is not None:
|
||||||
|
return val
|
||||||
|
return len(dut.rd_resp_row_ids) // len(dut.wr_addr)
|
||||||
|
|
||||||
|
|
||||||
|
def _pipeline_depth(num_rows, lanes):
|
||||||
|
"""Number of match-engine pipeline stages (rows processed per query)."""
|
||||||
|
return num_rows // lanes
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Bounded wait helper (RisingEdge only — keep the caller writeable)
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
async def _wait_bounded(dut, condition_fn, max_cycles, label):
|
||||||
|
"""Wait for *condition_fn* to become ``True``, sampling at each RisingEdge.
|
||||||
|
|
||||||
|
Raises ``AssertionError`` with *label* if *max_cycles* is exceeded.
|
||||||
|
The caller is left in a writeable phase after return.
|
||||||
|
"""
|
||||||
|
for _ in range(max_cycles):
|
||||||
|
await RisingEdge(dut.clk)
|
||||||
|
if condition_fn():
|
||||||
|
return
|
||||||
|
raise AssertionError(
|
||||||
|
f"{label}: timed out after {max_cycles} clock cycles"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _condition_signal_high(signal):
|
||||||
|
"""Return a nullary callable that checks *signal* is high (integer 1)."""
|
||||||
|
def _check():
|
||||||
|
return int(signal.value) == 1
|
||||||
|
return _check
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# DUT helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
async def reset_dut(dut):
|
||||||
|
"""Reset the DUT with new handshake interface.
|
||||||
|
|
||||||
|
``result_ready`` is held at 0 during measurement; the performance test
|
||||||
|
pulses it high for exactly one cycle after capturing ``result_valid``.
|
||||||
|
"""
|
||||||
|
dut.rst_n.value = 0
|
||||||
|
dut.wr_valid.value = 0
|
||||||
|
dut.wr_addr.value = 0
|
||||||
|
dut.write_hash.value = 0
|
||||||
|
dut.query_valid.value = 0
|
||||||
|
dut.query_hash.value = 0
|
||||||
|
dut.result_ready.value = 0
|
||||||
|
|
||||||
|
for _ in range(5):
|
||||||
|
await RisingEdge(dut.clk)
|
||||||
|
|
||||||
|
dut.rst_n.value = 1
|
||||||
|
|
||||||
|
for _ in range(2):
|
||||||
|
await RisingEdge(dut.clk)
|
||||||
|
|
||||||
|
|
||||||
|
async def wait_idle(dut, max_cycles=10):
|
||||||
|
"""Wait until both wr_ready=1 and query_ready=1 (system fully idle)."""
|
||||||
|
await _wait_bounded(
|
||||||
|
dut,
|
||||||
|
lambda: int(dut.wr_ready.value) and int(dut.query_ready.value),
|
||||||
|
max_cycles,
|
||||||
|
"wait_idle",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def write_row(dut, addr, value, max_cycles=10):
|
||||||
|
"""Write a single row using wr_valid/wr_ready handshake."""
|
||||||
|
await wait_idle(dut)
|
||||||
|
|
||||||
|
dut.wr_addr.value = addr
|
||||||
|
dut.write_hash.value = int(value)
|
||||||
|
dut.wr_valid.value = 1
|
||||||
|
|
||||||
|
await _wait_bounded(
|
||||||
|
dut,
|
||||||
|
_condition_signal_high(dut.wr_ready),
|
||||||
|
max_cycles,
|
||||||
|
f"write_row(addr={addr}) handshake",
|
||||||
|
)
|
||||||
|
|
||||||
|
dut.wr_valid.value = 0
|
||||||
|
await wait_idle(dut)
|
||||||
|
|
||||||
|
|
||||||
|
async def write_rows(dut, rows):
|
||||||
|
"""Write all rows sequentially."""
|
||||||
|
for idx, value in enumerate(rows):
|
||||||
|
await write_row(dut, idx, value)
|
||||||
|
|
||||||
|
|
||||||
|
async def query_once_with_latency(dut, query, max_result_cycles):
|
||||||
|
"""Issue a query and return (top1_index, top1_score, latency_cycles, total_cycles).
|
||||||
|
|
||||||
|
Parameters
|
||||||
|
----------
|
||||||
|
max_result_cycles : int
|
||||||
|
Hard bound on cycles from query acceptance to *result_valid*.
|
||||||
|
Derived from ``NUM_ROWS / LANES + pipeline_slack`` by the caller.
|
||||||
|
|
||||||
|
Cycle counting
|
||||||
|
--------------
|
||||||
|
``query_ready`` is a combinational signal that goes low *immediately* after
|
||||||
|
the RisingEdge at which the query is accepted (the state machine
|
||||||
|
transitions from S_IDLE to S_SCAN). We therefore sample ``query_ready``
|
||||||
|
**before** ``ReadOnly`` (i.e. at the RisingEdge time-point itself) to
|
||||||
|
capture the handshake.
|
||||||
|
|
||||||
|
``result_valid`` is a registered output that stays high until consumed, so
|
||||||
|
we sample it in the **settled phase** after ``ReadOnly``.
|
||||||
|
|
||||||
|
``latency_cycles`` is the number of RisingEdge events between the cycle
|
||||||
|
where the query is accepted and the cycle where ``result_valid`` is
|
||||||
|
observed.
|
||||||
|
"""
|
||||||
|
await wait_idle(dut)
|
||||||
|
|
||||||
|
edge_count = 0
|
||||||
|
dut.query_hash.value = int(query)
|
||||||
|
dut.query_valid.value = 1
|
||||||
|
|
||||||
|
# ── Phase 1: Accept handshake ───────────────────────────────────────
|
||||||
|
# query_ready is combinational — sample before ReadOnly.
|
||||||
|
accept_edge = None
|
||||||
|
for _ in range(10):
|
||||||
|
await RisingEdge(dut.clk)
|
||||||
|
edge_count += 1
|
||||||
|
q_ready = int(dut.query_ready.value) # sample before state transition
|
||||||
|
await ReadOnly()
|
||||||
|
await Timer(1, units="step") # exit ReadOnly for driving
|
||||||
|
if q_ready:
|
||||||
|
accept_edge = edge_count
|
||||||
|
break
|
||||||
|
assert accept_edge is not None, (
|
||||||
|
"Query accept handshake timed out after 10 cycles"
|
||||||
|
)
|
||||||
|
dut.query_valid.value = 0
|
||||||
|
|
||||||
|
# ── Phase 2: Wait for result_valid (measurement window) ─────────────
|
||||||
|
# result_valid is registered — sample after ReadOnly is fine.
|
||||||
|
result_edge = None
|
||||||
|
for _ in range(max_result_cycles):
|
||||||
|
await RisingEdge(dut.clk)
|
||||||
|
edge_count += 1
|
||||||
|
await ReadOnly()
|
||||||
|
if int(dut.result_valid.value):
|
||||||
|
result_edge = edge_count
|
||||||
|
break
|
||||||
|
assert result_edge is not None, (
|
||||||
|
f"Query result_valid timed out after {max_result_cycles} cycles "
|
||||||
|
f"(accepted at edge {accept_edge})"
|
||||||
|
)
|
||||||
|
await Timer(1, units="step")
|
||||||
|
|
||||||
|
latency_cycles = result_edge - accept_edge
|
||||||
|
|
||||||
|
top1_index = int(dut.top1_index.value)
|
||||||
|
top1_score = int(dut.top1_score.value)
|
||||||
|
|
||||||
|
# ── Phase 3: Consume result (pulse result_ready for one cycle) ──────
|
||||||
|
dut.result_ready.value = 1
|
||||||
|
await RisingEdge(dut.clk)
|
||||||
|
edge_count += 1
|
||||||
|
await ReadOnly()
|
||||||
|
await Timer(1, units="step")
|
||||||
|
dut.result_ready.value = 0
|
||||||
|
|
||||||
|
return top1_index, top1_score, latency_cycles, edge_count
|
||||||
|
|
||||||
|
|
||||||
|
def deterministic_rows(num_rows, hash_bits, query_hash):
|
||||||
|
"""Create deterministic rows where only row 0 stores *query_hash*."""
|
||||||
|
mask = (1 << hash_bits) - 1
|
||||||
|
rows = [0] * num_rows
|
||||||
|
rows[0] = query_hash
|
||||||
|
for i in range(1, num_rows):
|
||||||
|
# Deterministic non-matching value; golden-ratio-like spread
|
||||||
|
val = ((i + 1) * 0x9E3779B97F4A7C15) & mask
|
||||||
|
if val == query_hash or val == 0:
|
||||||
|
val = ((val ^ query_hash) ^ 0xA5A5A5A5A5A5A5A5) & mask
|
||||||
|
if val == query_hash or val == 0:
|
||||||
|
val = (val ^ 1) & mask
|
||||||
|
rows[i] = val
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
# ── Performance Test ──────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
@cocotb.test()
|
||||||
|
async def cam_perf_benchmark(dut):
|
||||||
|
"""Performance benchmark: measure query latency in cycles."""
|
||||||
|
cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start())
|
||||||
|
await reset_dut(dut)
|
||||||
|
|
||||||
|
num_rows = dut_num_rows(dut)
|
||||||
|
hash_bits = dut_hash_bits(dut)
|
||||||
|
lanes = dut_lanes(dut)
|
||||||
|
write_noise_en = _get_param(dut, "WRITE_NOISE_EN", 1)
|
||||||
|
read_noise_en = _get_param(dut, "READ_NOISE_EN", 0)
|
||||||
|
|
||||||
|
# ── Deterministic query ─────────────────────────────────────────────
|
||||||
|
query_hash = 0xAA55_AA55_AA55_AA55_AA55_AA55_AA55_AA55
|
||||||
|
query_hash &= (1 << hash_bits) - 1
|
||||||
|
if query_hash == 0:
|
||||||
|
query_hash = 1
|
||||||
|
|
||||||
|
rows = deterministic_rows(num_rows, hash_bits, query_hash)
|
||||||
|
await write_rows(dut, rows)
|
||||||
|
|
||||||
|
# Bound: pipeline depth plus generous slack for read + drain stages
|
||||||
|
pipeline = _pipeline_depth(num_rows, lanes)
|
||||||
|
max_result_cycles = pipeline + 30
|
||||||
|
|
||||||
|
top1_index, top1_score, latency_cycles, total_cycles = (
|
||||||
|
await query_once_with_latency(dut, query_hash, max_result_cycles)
|
||||||
|
)
|
||||||
|
|
||||||
|
# ── Performance assertions ──────────────────────────────────────────
|
||||||
|
assert latency_cycles > 0, (
|
||||||
|
f"latency_cycles must be positive, got {latency_cycles}"
|
||||||
|
)
|
||||||
|
assert total_cycles > 0, (
|
||||||
|
f"total_cycles must be positive, got {total_cycles}"
|
||||||
|
)
|
||||||
|
|
||||||
|
# ── Correctness assertions (conditional on noise state) ─────────────
|
||||||
|
if not write_noise_en and not read_noise_en:
|
||||||
|
# Without noise: stored hash at row 0 == query_hash → exact match.
|
||||||
|
assert top1_index == 0, (
|
||||||
|
f"Noise disabled: expected top1_index=0 (exact match), got "
|
||||||
|
f"{top1_index}"
|
||||||
|
)
|
||||||
|
assert top1_score == hash_bits, (
|
||||||
|
f"Noise disabled: expected top1_score={hash_bits}, got "
|
||||||
|
f"{top1_score}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
# With noise: write/read flip masks may corrupt stored values, so
|
||||||
|
# we cannot reliably assert the exact match. Instead, confirm a
|
||||||
|
# valid non-zero score was produced (the match engine ran).
|
||||||
|
assert top1_score > 0, (
|
||||||
|
f"Noise enabled: expected top1_score > 0, got {top1_score}. "
|
||||||
|
"Match engine returned invalid result."
|
||||||
|
)
|
||||||
|
dut._log.info(
|
||||||
|
"Noise enabled (WRITE_NOISE_EN=%s, READ_NOISE_EN=%s) — "
|
||||||
|
"skipping exact top1_index/top1_score assertion. "
|
||||||
|
"top1_index=%d top1_score=%d",
|
||||||
|
write_noise_en, read_noise_en, top1_index, top1_score,
|
||||||
|
)
|
||||||
|
|
||||||
|
# ── Machine-readable performance marker ─────────────────────────────
|
||||||
|
queries_per_cycle = 1.0 / total_cycles
|
||||||
|
dut._log.info(
|
||||||
|
"PERF_RESULT latency_cycles=%d total_cycles=%d "
|
||||||
|
"accepted_queries=1 completed_queries=1 "
|
||||||
|
"queries_per_cycle=%.6f status=pass",
|
||||||
|
latency_cycles, total_cycles, queries_per_cycle,
|
||||||
|
)
|
||||||
762
scripts/sweep_cam_perf.py
Normal file
762
scripts/sweep_cam_perf.py
Normal file
@@ -0,0 +1,762 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""CAM performance simulation sweep — data model, sweep matrix, and command builders."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import csv
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import subprocess
|
||||||
|
import time
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import date as date_type
|
||||||
|
from datetime import datetime
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Annotated
|
||||||
|
|
||||||
|
import typer
|
||||||
|
|
||||||
|
# Timeout for each external simulation invocation (30 minutes).
|
||||||
|
DEFAULT_COMMAND_TIMEOUT_SEC = 1800
|
||||||
|
|
||||||
|
# Cocotb test module that records PERF_RESULT metrics.
|
||||||
|
PERF_MODULE = "tests.test_cam_perf"
|
||||||
|
|
||||||
|
# CSV column names for sweep result output — must match exactly.
|
||||||
|
CSV_FIELDNAMES = [
|
||||||
|
"sweep_name",
|
||||||
|
"num_rows",
|
||||||
|
"hash_bits",
|
||||||
|
"lanes",
|
||||||
|
"latency_cycles",
|
||||||
|
"total_cycles",
|
||||||
|
"accepted_queries",
|
||||||
|
"completed_queries",
|
||||||
|
"queries_per_cycle",
|
||||||
|
"sim_time_sec",
|
||||||
|
"status",
|
||||||
|
"log_path",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Dense sweep-matrix value sets for paper-quality figures.
|
||||||
|
NUM_ROWS_SWEEP_VALUES = (64, 128, 192, 256, 384, 512, 768, 1024)
|
||||||
|
HASH_BITS_SWEEP_VALUES = (64, 128, 192, 256, 320, 384, 448, 512)
|
||||||
|
LANES_SWEEP_VALUES = (2, 4, 8, 16, 32)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Data model
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class SweepConfig:
|
||||||
|
"""A single point in the CAM performance sweep matrix."""
|
||||||
|
|
||||||
|
sweep_name: str
|
||||||
|
num_rows: int
|
||||||
|
hash_bits: int
|
||||||
|
lanes: int
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class PerfRun:
|
||||||
|
"""Result of executing a single sweep configuration."""
|
||||||
|
|
||||||
|
config: SweepConfig
|
||||||
|
status: str
|
||||||
|
returncode: int
|
||||||
|
duration_sec: float
|
||||||
|
log_path: Path
|
||||||
|
metrics: dict[str, str]
|
||||||
|
error: str = ""
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Default sweep matrix
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def default_sweep_configs() -> list[SweepConfig]:
|
||||||
|
"""Return the paper-supporting dense sweep matrix (19 unique configs).
|
||||||
|
|
||||||
|
The matrix sweeps one dimension at a time:
|
||||||
|
* NUM_ROWS: ``(nr, 512, 16)`` for ``nr`` in ``NUM_ROWS_SWEEP_VALUES``
|
||||||
|
* HASH_BITS: ``(512, hb, 16)`` for ``hb`` in ``HASH_BITS_SWEEP_VALUES``
|
||||||
|
* LANES: ``(512, 512, la)`` for ``la`` in ``LANES_SWEEP_VALUES``
|
||||||
|
|
||||||
|
Overlapping triples are deduplicated preserving first occurrence.
|
||||||
|
"""
|
||||||
|
candidates = (
|
||||||
|
[SweepConfig("num_rows", nr, 512, 16) for nr in NUM_ROWS_SWEEP_VALUES]
|
||||||
|
+ [SweepConfig("hash_bits", 512, hb, 16) for hb in HASH_BITS_SWEEP_VALUES]
|
||||||
|
+ [SweepConfig("lanes", 512, 512, la) for la in LANES_SWEEP_VALUES]
|
||||||
|
)
|
||||||
|
seen: set[tuple[int, int, int]] = set()
|
||||||
|
result: list[SweepConfig] = []
|
||||||
|
for cfg in candidates:
|
||||||
|
key = (cfg.num_rows, cfg.hash_bits, cfg.lanes)
|
||||||
|
if key not in seen:
|
||||||
|
seen.add(key)
|
||||||
|
result.append(cfg)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Sweep membership helper
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def logical_sweeps_for_config(config: SweepConfig) -> tuple[str, ...]:
|
||||||
|
"""Return the logical sweep-group memberships for *config*.
|
||||||
|
|
||||||
|
A single config may participate in multiple sweep dimensions. This is
|
||||||
|
especially important for the baseline config ``(512, 512, 16)`` whose
|
||||||
|
singleton CSV row has ``sweep_name='num_rows'`` (first occurrence wins
|
||||||
|
during dedup), yet logically belongs to **all three** sweep groups.
|
||||||
|
|
||||||
|
The predicates reference the module-level dense-value constants:
|
||||||
|
* ``'num_rows'`` – ``HASH_BITS=512, LANES=16, NUM_ROWS ∈ NUM_ROWS_SWEEP_VALUES``
|
||||||
|
* ``'hash_bits'`` – ``NUM_ROWS=512, LANES=16, HASH_BITS ∈ HASH_BITS_SWEEP_VALUES``
|
||||||
|
* ``'lanes'`` – ``NUM_ROWS=512, HASH_BITS=512, LANES ∈ LANES_SWEEP_VALUES``
|
||||||
|
|
||||||
|
Downstream plot / analysis code **must** use this helper to derive sweep
|
||||||
|
groups instead of filtering solely by ``SweepConfig.sweep_name``; otherwise
|
||||||
|
the baseline ``(512,512,16)`` would be missing from ``hash_bits`` and
|
||||||
|
``lanes`` group queries.
|
||||||
|
"""
|
||||||
|
memberships: list[str] = []
|
||||||
|
nr, hb, la = config.num_rows, config.hash_bits, config.lanes
|
||||||
|
|
||||||
|
if hb == 512 and la == 16 and nr in NUM_ROWS_SWEEP_VALUES:
|
||||||
|
memberships.append("num_rows")
|
||||||
|
if nr == 512 and la == 16 and hb in HASH_BITS_SWEEP_VALUES:
|
||||||
|
memberships.append("hash_bits")
|
||||||
|
if nr == 512 and hb == 512 and la in LANES_SWEEP_VALUES:
|
||||||
|
memberships.append("lanes")
|
||||||
|
|
||||||
|
return tuple(memberships)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Log parsing
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
_PERF_EXPECTED_KEYS = frozenset({
|
||||||
|
"latency_cycles",
|
||||||
|
"total_cycles",
|
||||||
|
"accepted_queries",
|
||||||
|
"completed_queries",
|
||||||
|
"queries_per_cycle",
|
||||||
|
"status",
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def parse_perf_result(log_text: str) -> dict[str, str]:
|
||||||
|
"""Parse the first *complete* ``PERF_RESULT`` marker line from a simulation log.
|
||||||
|
|
||||||
|
Scans all lines for a ``PERF_RESULT`` marker (standalone word). For each
|
||||||
|
such line, extracts ``key=value`` tokens and checks whether all expected
|
||||||
|
keys (``latency_cycles``, ``total_cycles``, ``accepted_queries``,
|
||||||
|
``completed_queries``, ``queries_per_cycle``, ``status``) are present.
|
||||||
|
Returns the dict from the **first** line with a complete set, or ``{}``
|
||||||
|
if no complete marker exists in the entire log.
|
||||||
|
"""
|
||||||
|
_marker_re = re.compile(r"\bPERF_RESULT\b")
|
||||||
|
for line in log_text.splitlines():
|
||||||
|
if _marker_re.search(line):
|
||||||
|
tokens = line.split()
|
||||||
|
parsed: dict[str, str] = {}
|
||||||
|
for token in tokens:
|
||||||
|
if "=" in token:
|
||||||
|
key, _, value = token.partition("=")
|
||||||
|
parsed[key] = value
|
||||||
|
if _PERF_EXPECTED_KEYS.issubset(parsed.keys()):
|
||||||
|
return parsed
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Command builders
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def build_make_command(config: SweepConfig) -> list[str]:
|
||||||
|
"""Build a ``make`` command to run the perf test module with *config*."""
|
||||||
|
return [
|
||||||
|
"make",
|
||||||
|
"-C",
|
||||||
|
"hw/sim",
|
||||||
|
f"NUM_ROWS={config.num_rows}",
|
||||||
|
f"HASH_BITS={config.hash_bits}",
|
||||||
|
f"LANES={config.lanes}",
|
||||||
|
f"COCOTB_TEST_MODULES={PERF_MODULE}",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Execution helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _run_subprocess(
|
||||||
|
cmd: list[str],
|
||||||
|
*,
|
||||||
|
cwd: Path,
|
||||||
|
env: dict[str, str],
|
||||||
|
timeout: int,
|
||||||
|
) -> tuple[subprocess.CompletedProcess[str] | None, float, str | None]:
|
||||||
|
"""Execute *cmd*, capture output, enforce *timeout*.
|
||||||
|
|
||||||
|
Returns ``(result, duration_sec, error_msg)``.
|
||||||
|
On success *result* is a ``CompletedProcess`` and *error_msg* is ``None``.
|
||||||
|
On failure (timeout / OSError / SubprocessError) *result* is ``None``
|
||||||
|
and *error_msg* describes what happened.
|
||||||
|
"""
|
||||||
|
start = time.monotonic()
|
||||||
|
try:
|
||||||
|
result = subprocess.run(
|
||||||
|
cmd,
|
||||||
|
cwd=cwd,
|
||||||
|
env=env,
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
except subprocess.TimeoutExpired as exc:
|
||||||
|
end = time.monotonic()
|
||||||
|
partial = ""
|
||||||
|
if exc.stdout:
|
||||||
|
partial += f"\n# [partial stdout before timeout]\n{exc.stdout}"
|
||||||
|
if exc.stderr:
|
||||||
|
partial += f"\n# [partial stderr before timeout]\n{exc.stderr}"
|
||||||
|
return None, round(end - start, 3), f"timed out after {timeout}s{partial}"
|
||||||
|
except (OSError, subprocess.SubprocessError) as exc:
|
||||||
|
end = time.monotonic()
|
||||||
|
return None, round(end - start, 3), str(exc)
|
||||||
|
end = time.monotonic()
|
||||||
|
return result, round(end - start, 3), None
|
||||||
|
|
||||||
|
|
||||||
|
def run_one_config(
|
||||||
|
config: SweepConfig,
|
||||||
|
*,
|
||||||
|
output_dir: Path,
|
||||||
|
repo_root: Path,
|
||||||
|
timeout: int = DEFAULT_COMMAND_TIMEOUT_SEC,
|
||||||
|
) -> PerfRun:
|
||||||
|
"""Execute a single sweep configuration and return the result.
|
||||||
|
|
||||||
|
Steps:
|
||||||
|
1. Create logs directory.
|
||||||
|
2. Validate ``NUM_ROWS % LANES == 0``; return fail immediately if invalid.
|
||||||
|
3. Run ``make -C hw/sim clean``.
|
||||||
|
4. Build and run the simulation command via ``build_make_command(config)``.
|
||||||
|
5. Capture combined stdout/stderr into a log file named
|
||||||
|
``<sweep_name>_<num_rows>r_<hash_bits>b_<lanes>l.log``.
|
||||||
|
6. Parse ``PERF_RESULT`` from the combined log.
|
||||||
|
7. Determine pass/fail: pass only if returncode == 0 and metrics status == 'pass'.
|
||||||
|
"""
|
||||||
|
logs_dir = output_dir / "logs"
|
||||||
|
logs_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
log_filename = (
|
||||||
|
f"{config.sweep_name}_{config.num_rows}r_{config.hash_bits}b_{config.lanes}l.log"
|
||||||
|
)
|
||||||
|
log_path = logs_dir / log_filename
|
||||||
|
relative_log = Path("logs") / log_filename
|
||||||
|
|
||||||
|
# ── Validation: NUM_ROWS must be divisible by LANES ──────────────────
|
||||||
|
if config.num_rows % config.lanes != 0:
|
||||||
|
log_path.write_text(
|
||||||
|
f"# ERROR: NUM_ROWS ({config.num_rows}) not divisible by LANES ({config.lanes})\n"
|
||||||
|
)
|
||||||
|
return PerfRun(
|
||||||
|
config=config,
|
||||||
|
status="fail",
|
||||||
|
returncode=-1,
|
||||||
|
duration_sec=0.0,
|
||||||
|
log_path=relative_log,
|
||||||
|
metrics={},
|
||||||
|
error=f"NUM_ROWS ({config.num_rows}) not divisible by LANES ({config.lanes})",
|
||||||
|
)
|
||||||
|
|
||||||
|
env = os.environ.copy()
|
||||||
|
log_chunks: list[str] = []
|
||||||
|
|
||||||
|
# ── Clean step ───────────────────────────────────────────────────────
|
||||||
|
clean_cmd = ["make", "-C", "hw/sim", "clean"]
|
||||||
|
log_chunks.append(f"# Command: {' '.join(clean_cmd)}\n")
|
||||||
|
clean_result, clean_duration, clean_err = _run_subprocess(
|
||||||
|
clean_cmd, cwd=repo_root, env=env, timeout=timeout,
|
||||||
|
)
|
||||||
|
if clean_result is not None:
|
||||||
|
log_chunks.append(clean_result.stdout)
|
||||||
|
log_chunks.append(clean_result.stderr)
|
||||||
|
else:
|
||||||
|
log_chunks.append(f"# ERROR: {clean_err}\n")
|
||||||
|
|
||||||
|
if clean_err is not None or (
|
||||||
|
clean_result is not None and clean_result.returncode != 0
|
||||||
|
):
|
||||||
|
returncode = clean_result.returncode if clean_result is not None else -1
|
||||||
|
log_path.write_text("".join(log_chunks))
|
||||||
|
return PerfRun(
|
||||||
|
config=config,
|
||||||
|
status="fail",
|
||||||
|
returncode=returncode,
|
||||||
|
duration_sec=clean_duration,
|
||||||
|
log_path=relative_log,
|
||||||
|
metrics={},
|
||||||
|
error=clean_err or f"make clean failed with returncode {returncode}",
|
||||||
|
)
|
||||||
|
|
||||||
|
# ── Main simulation command ──────────────────────────────────────────
|
||||||
|
command = build_make_command(config)
|
||||||
|
log_chunks.append(f"# Command: {' '.join(command)}\n")
|
||||||
|
main_result, duration, main_err = _run_subprocess(
|
||||||
|
command, cwd=repo_root, env=env, timeout=timeout,
|
||||||
|
)
|
||||||
|
|
||||||
|
if main_result is not None:
|
||||||
|
log_chunks.append(main_result.stdout)
|
||||||
|
log_chunks.append(main_result.stderr)
|
||||||
|
else:
|
||||||
|
log_chunks.append(f"# ERROR: {main_err}\n")
|
||||||
|
|
||||||
|
combined_log = "".join(log_chunks)
|
||||||
|
log_path.write_text(combined_log)
|
||||||
|
|
||||||
|
# ── Parse PERF_RESULT ────────────────────────────────────────────────
|
||||||
|
metrics = parse_perf_result(combined_log)
|
||||||
|
|
||||||
|
if main_err is not None:
|
||||||
|
status = "fail"
|
||||||
|
returncode = -1
|
||||||
|
error = main_err
|
||||||
|
else:
|
||||||
|
returncode = main_result.returncode
|
||||||
|
metrics_status = metrics.get("status", "")
|
||||||
|
if returncode == 0 and metrics_status == "pass":
|
||||||
|
status = "pass"
|
||||||
|
error = ""
|
||||||
|
else:
|
||||||
|
status = "fail"
|
||||||
|
if returncode != 0:
|
||||||
|
error = f"simulation returned non-zero exit code {returncode}"
|
||||||
|
elif metrics_status != "pass":
|
||||||
|
error = f"PERF_RESULT status is '{metrics_status}'"
|
||||||
|
else:
|
||||||
|
error = "unknown failure"
|
||||||
|
|
||||||
|
return PerfRun(
|
||||||
|
config=config,
|
||||||
|
status=status,
|
||||||
|
returncode=returncode,
|
||||||
|
duration_sec=duration,
|
||||||
|
log_path=relative_log,
|
||||||
|
metrics=metrics,
|
||||||
|
error=error,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# CSV helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def run_to_row(run: PerfRun) -> dict[str, str]:
|
||||||
|
"""Convert a ``PerfRun`` to a CSV row dict matching ``CSV_FIELDNAMES``."""
|
||||||
|
return {
|
||||||
|
"sweep_name": run.config.sweep_name,
|
||||||
|
"num_rows": str(run.config.num_rows),
|
||||||
|
"hash_bits": str(run.config.hash_bits),
|
||||||
|
"lanes": str(run.config.lanes),
|
||||||
|
"latency_cycles": run.metrics.get("latency_cycles", ""),
|
||||||
|
"total_cycles": run.metrics.get("total_cycles", ""),
|
||||||
|
"accepted_queries": run.metrics.get("accepted_queries", ""),
|
||||||
|
"completed_queries": run.metrics.get("completed_queries", ""),
|
||||||
|
"queries_per_cycle": run.metrics.get("queries_per_cycle", ""),
|
||||||
|
"sim_time_sec": str(run.duration_sec),
|
||||||
|
"status": run.status,
|
||||||
|
"log_path": str(run.log_path),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def write_csv(output_dir: Path, runs: list[PerfRun]) -> None:
|
||||||
|
"""Write ``sweep.csv`` with ``CSV_FIELDNAMES`` header and one row per run."""
|
||||||
|
path = output_dir / "sweep.csv"
|
||||||
|
with open(path, "w", newline="") as f:
|
||||||
|
writer = csv.DictWriter(f, fieldnames=CSV_FIELDNAMES)
|
||||||
|
writer.writeheader()
|
||||||
|
for run in runs:
|
||||||
|
writer.writerow(run_to_row(run))
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Output helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _float_or_none(value: str) -> float | None:
|
||||||
|
"""Parse *value* as ``float``, returning ``None`` on failure."""
|
||||||
|
try:
|
||||||
|
return float(value)
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Nature-like plot style constants
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
PLOT_NAVY = "#243B53"
|
||||||
|
PLOT_SLATE = "#7A8793"
|
||||||
|
PLOT_TEAL = "#4FA7A3"
|
||||||
|
PLOT_GRID = "#D7DEE5"
|
||||||
|
PLOT_TEXT = "#1F2933"
|
||||||
|
|
||||||
|
|
||||||
|
def _style_perf_axis(ax, *, title: str, xlabel: str, ylabel: str) -> None:
|
||||||
|
"""Apply Nature-like styling to a performance figure axis."""
|
||||||
|
ax.set_title(title, fontsize=10, color=PLOT_TEXT, pad=8)
|
||||||
|
ax.set_xlabel(xlabel, fontsize=9, color=PLOT_TEXT)
|
||||||
|
ax.set_ylabel(ylabel, fontsize=9, color=PLOT_TEXT)
|
||||||
|
ax.grid(True, color=PLOT_GRID, linewidth=0.6, alpha=0.8)
|
||||||
|
ax.spines["top"].set_visible(False)
|
||||||
|
ax.spines["right"].set_visible(False)
|
||||||
|
ax.spines["left"].set_color(PLOT_SLATE)
|
||||||
|
ax.spines["bottom"].set_color(PLOT_SLATE)
|
||||||
|
ax.tick_params(axis="both", colors=PLOT_TEXT, labelsize=8, width=0.6)
|
||||||
|
|
||||||
|
|
||||||
|
def _plot_points(
|
||||||
|
runs: list[PerfRun], group: str, x_getter, metric: str,
|
||||||
|
) -> list[tuple[int, float]]:
|
||||||
|
"""Filter *runs* to pass runs in *group* and return sorted ``(x, y)`` points."""
|
||||||
|
points: list[tuple[int, float]] = []
|
||||||
|
for run in runs:
|
||||||
|
if run.status != "pass" or group not in logical_sweeps_for_config(run.config):
|
||||||
|
continue
|
||||||
|
value = _float_or_none(run.metrics.get(metric, ""))
|
||||||
|
if value is None:
|
||||||
|
continue
|
||||||
|
points.append((int(x_getter(run.config)), value))
|
||||||
|
return sorted(points, key=lambda item: item[0])
|
||||||
|
|
||||||
|
|
||||||
|
def write_summary(output_dir: Path, runs: list[PerfRun]) -> None:
|
||||||
|
"""Write ``sweep_summary.md`` in Chinese with summary table and disclaimer."""
|
||||||
|
total = len(runs)
|
||||||
|
passed = sum(1 for r in runs if r.status == "pass")
|
||||||
|
failed = total - passed
|
||||||
|
failed_runs = [r for r in runs if r.status == "fail"]
|
||||||
|
|
||||||
|
lines: list[str] = []
|
||||||
|
lines.append("# CAM 性能扫描摘要")
|
||||||
|
lines.append("")
|
||||||
|
lines.append(f"**输出目录:** {output_dir}")
|
||||||
|
lines.append(f"**总运行数:** {total}")
|
||||||
|
lines.append(f"**通过数:** {passed}")
|
||||||
|
lines.append(f"**失败数:** {failed}")
|
||||||
|
lines.append("")
|
||||||
|
lines.append("## 性能结果")
|
||||||
|
lines.append("")
|
||||||
|
lines.append("| 扫描分组 | NUM_ROWS | HASH_BITS | LANES | 延迟(周期数) | 查询量/周期 | 状态 | 日志 |")
|
||||||
|
lines.append("|----------|----------|-----------|-------|-------------|-------------|------|------|")
|
||||||
|
for r in runs:
|
||||||
|
lines.append(
|
||||||
|
f"| {r.config.sweep_name} | {r.config.num_rows} | {r.config.hash_bits} "
|
||||||
|
f"| {r.config.lanes} | {r.metrics.get('latency_cycles', '')} "
|
||||||
|
f"| {r.metrics.get('queries_per_cycle', '')} | {r.status} "
|
||||||
|
f"| {r.log_path} |"
|
||||||
|
)
|
||||||
|
lines.append("")
|
||||||
|
lines.append("## 失败列表")
|
||||||
|
lines.append("")
|
||||||
|
if failed_runs:
|
||||||
|
for r in failed_runs:
|
||||||
|
err = r.error or "无"
|
||||||
|
lines.append(f"- **{r.config.sweep_name}** (`{r.log_path}`): {err}")
|
||||||
|
else:
|
||||||
|
lines.append("- 无")
|
||||||
|
lines.append("")
|
||||||
|
lines.append("## 免责声明")
|
||||||
|
lines.append("")
|
||||||
|
lines.append(
|
||||||
|
"本报告为 **Verilator/Cocotb 仿真** 结果,非 FPGA 板级实测性能、"
|
||||||
|
"资源利用率或 Fmax。所有数据仅供架构评估参考,不代表最终 FPGA 实现指标。"
|
||||||
|
)
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
path = output_dir / "sweep_summary.md"
|
||||||
|
path.write_text("\n".join(lines))
|
||||||
|
|
||||||
|
|
||||||
|
def write_notes(output_dir: Path, runs: list[PerfRun]) -> None:
|
||||||
|
"""Write ``ch6_3_cam_perf_notes.md`` in Chinese with trend analysis."""
|
||||||
|
lines: list[str] = []
|
||||||
|
lines.append("# 第 6.3 节 CAM 性能仿真笔记")
|
||||||
|
lines.append("")
|
||||||
|
lines.append("## 说明")
|
||||||
|
lines.append("")
|
||||||
|
lines.append(
|
||||||
|
"以下结果基于 **Verilator/Cocotb 仿真** 环境,旨在评估 CAM 设计的查询延迟"
|
||||||
|
"(latency_cycles)和吞吐量(queries_per_cycle)趋势。所有数据均为仿真"
|
||||||
|
"结果,**非 FPGA 板级实测性能**。本文档不提供 LUT、FF、BRAM 或 Fmax 等"
|
||||||
|
"资源利用率或时序结论。"
|
||||||
|
)
|
||||||
|
lines.append("")
|
||||||
|
lines.append("## 参数趋势分析")
|
||||||
|
lines.append("")
|
||||||
|
lines.append("### NUM_ROWS(CAM 行数)")
|
||||||
|
lines.append("")
|
||||||
|
nr_runs = [
|
||||||
|
r for r in runs
|
||||||
|
if "num_rows" in logical_sweeps_for_config(r.config) and r.status == "pass"
|
||||||
|
]
|
||||||
|
if nr_runs:
|
||||||
|
for r in sorted(nr_runs, key=lambda x: x.config.num_rows):
|
||||||
|
lat = r.metrics.get("latency_cycles", "N/A")
|
||||||
|
qpc = r.metrics.get("queries_per_cycle", "N/A")
|
||||||
|
lines.append(f"- NUM_ROWS={r.config.num_rows}: 延迟={lat} 周期, 吞吐量={qpc} 查询/周期")
|
||||||
|
else:
|
||||||
|
lines.append("- 无通过数据")
|
||||||
|
lines.append("")
|
||||||
|
lines.append("### HASH_BITS(哈希位宽)")
|
||||||
|
lines.append("")
|
||||||
|
hb_runs = [
|
||||||
|
r for r in runs
|
||||||
|
if "hash_bits" in logical_sweeps_for_config(r.config) and r.status == "pass"
|
||||||
|
]
|
||||||
|
if hb_runs:
|
||||||
|
for r in sorted(hb_runs, key=lambda x: x.config.hash_bits):
|
||||||
|
lat = r.metrics.get("latency_cycles", "N/A")
|
||||||
|
qpc = r.metrics.get("queries_per_cycle", "N/A")
|
||||||
|
lines.append(f"- HASH_BITS={r.config.hash_bits}: 延迟={lat} 周期, 吞吐量={qpc} 查询/周期")
|
||||||
|
else:
|
||||||
|
lines.append("- 无通过数据")
|
||||||
|
lines.append("")
|
||||||
|
lines.append("### LANES(流水线通路数)")
|
||||||
|
lines.append("")
|
||||||
|
la_runs = [
|
||||||
|
r for r in runs
|
||||||
|
if "lanes" in logical_sweeps_for_config(r.config) and r.status == "pass"
|
||||||
|
]
|
||||||
|
if la_runs:
|
||||||
|
for r in sorted(la_runs, key=lambda x: x.config.lanes):
|
||||||
|
lat = r.metrics.get("latency_cycles", "N/A")
|
||||||
|
qpc = r.metrics.get("queries_per_cycle", "N/A")
|
||||||
|
lines.append(f"- LANES={r.config.lanes}: 延迟={lat} 周期, 吞吐量={qpc} 查询/周期")
|
||||||
|
else:
|
||||||
|
lines.append("- 无通过数据")
|
||||||
|
lines.append("")
|
||||||
|
lines.append("## 免责声明")
|
||||||
|
lines.append("")
|
||||||
|
lines.append(
|
||||||
|
"本文档中的数据来源于 Verilator/Cocotb 仿真,用于观察性能趋势。"
|
||||||
|
"实际 FPGA 实现的性能、资源利用率和最大工作频率(Fmax)可能因综合选项、"
|
||||||
|
"布局布线、芯片型号等因素而显著不同。请勿将本文档中的数据直接用作 FPGA "
|
||||||
|
"实现指标。"
|
||||||
|
)
|
||||||
|
lines.append("")
|
||||||
|
|
||||||
|
path = output_dir / "ch6_3_cam_perf_notes.md"
|
||||||
|
path.write_text("\n".join(lines))
|
||||||
|
|
||||||
|
|
||||||
|
def plot_figures(output_dir: Path, runs: list[PerfRun]) -> None:
|
||||||
|
"""Generate performance trend figures under ``figures/``.
|
||||||
|
|
||||||
|
Three single-panel PNGs and one 1×3 multi-panel PNG are produced
|
||||||
|
when at least 2 pass rows exist in a logical sweep group:
|
||||||
|
- ``cam_num_rows_vs_latency.png``
|
||||||
|
- ``cam_hash_bits_vs_latency.png``
|
||||||
|
- ``cam_lanes_vs_throughput.png``
|
||||||
|
- ``cam_perf_multipanel.png``
|
||||||
|
|
||||||
|
Uses ``logical_sweeps_for_config`` so the baseline ``(512,512,16)``
|
||||||
|
appears in all three groups.
|
||||||
|
"""
|
||||||
|
pass_runs = [r for r in runs if r.status == "pass"]
|
||||||
|
if not pass_runs:
|
||||||
|
return
|
||||||
|
|
||||||
|
import matplotlib
|
||||||
|
matplotlib.use("Agg")
|
||||||
|
import matplotlib.pyplot as plt
|
||||||
|
|
||||||
|
# White / Nature-like rcParams defaults.
|
||||||
|
plt.rcParams.update({
|
||||||
|
"figure.facecolor": "white",
|
||||||
|
"axes.facecolor": "white",
|
||||||
|
"savefig.facecolor": "white",
|
||||||
|
"font.family": "DejaVu Sans",
|
||||||
|
"axes.linewidth": 0.8,
|
||||||
|
})
|
||||||
|
|
||||||
|
fig_dir = output_dir / "figures"
|
||||||
|
fig_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
# ── Define the three logical groups ──────────────────────────────────
|
||||||
|
groups: list[dict] = [
|
||||||
|
{
|
||||||
|
"group": "num_rows",
|
||||||
|
"x_getter": lambda c: c.num_rows,
|
||||||
|
"metric": "latency_cycles",
|
||||||
|
"xlabel": "NUM_ROWS",
|
||||||
|
"ylabel": "Latency (cycles)",
|
||||||
|
"title": "CAM Query Latency vs NUM_ROWS\n(HASH_BITS=512, LANES=16)",
|
||||||
|
"filename": "cam_num_rows_vs_latency.png",
|
||||||
|
"marker": "o",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"group": "hash_bits",
|
||||||
|
"x_getter": lambda c: c.hash_bits,
|
||||||
|
"metric": "latency_cycles",
|
||||||
|
"xlabel": "HASH_BITS",
|
||||||
|
"ylabel": "Latency (cycles)",
|
||||||
|
"title": "CAM Query Latency vs HASH_BITS\n(NUM_ROWS=512, LANES=16)",
|
||||||
|
"filename": "cam_hash_bits_vs_latency.png",
|
||||||
|
"marker": "s",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"group": "lanes",
|
||||||
|
"x_getter": lambda c: c.lanes,
|
||||||
|
"metric": "queries_per_cycle",
|
||||||
|
"xlabel": "LANES",
|
||||||
|
"ylabel": "Queries / Cycle",
|
||||||
|
"title": "CAM Throughput vs LANES\n(NUM_ROWS=512, HASH_BITS=512)",
|
||||||
|
"filename": "cam_lanes_vs_throughput.png",
|
||||||
|
"marker": "^",
|
||||||
|
},
|
||||||
|
]
|
||||||
|
|
||||||
|
# ── Individual figures ───────────────────────────────────────────────
|
||||||
|
for g in groups:
|
||||||
|
points = _plot_points(pass_runs, g["group"], g["x_getter"], g["metric"])
|
||||||
|
if len(points) < 2:
|
||||||
|
continue
|
||||||
|
xs, ys = zip(*points)
|
||||||
|
fig, ax = plt.subplots(figsize=(4.2, 3.0))
|
||||||
|
ax.plot(
|
||||||
|
xs, ys,
|
||||||
|
color=PLOT_NAVY, linewidth=1.6,
|
||||||
|
marker=g["marker"], markersize=4.5,
|
||||||
|
markerfacecolor=PLOT_TEAL, markeredgecolor="white", markeredgewidth=0.7,
|
||||||
|
)
|
||||||
|
_style_perf_axis(ax, title=g["title"], xlabel=g["xlabel"], ylabel=g["ylabel"])
|
||||||
|
fig.tight_layout(pad=0.7)
|
||||||
|
fig.savefig(fig_dir / g["filename"], dpi=300, bbox_inches="tight")
|
||||||
|
plt.close(fig)
|
||||||
|
|
||||||
|
# ── Multi-panel figure (1 × 3) ───────────────────────────────────────
|
||||||
|
fig, axes = plt.subplots(1, 3, figsize=(10.8, 3.0), constrained_layout=True)
|
||||||
|
for ax, g in zip(axes, groups):
|
||||||
|
points = _plot_points(pass_runs, g["group"], g["x_getter"], g["metric"])
|
||||||
|
if len(points) < 2:
|
||||||
|
ax.text(0.5, 0.5, "insufficient data", ha="center", va="center",
|
||||||
|
transform=ax.transAxes, fontsize=8, color=PLOT_SLATE)
|
||||||
|
continue
|
||||||
|
xs, ys = zip(*points)
|
||||||
|
ax.plot(
|
||||||
|
xs, ys,
|
||||||
|
color=PLOT_NAVY, linewidth=1.6,
|
||||||
|
marker=g["marker"], markersize=4.5,
|
||||||
|
markerfacecolor=PLOT_TEAL, markeredgecolor="white", markeredgewidth=0.7,
|
||||||
|
)
|
||||||
|
_style_perf_axis(ax, title=g["title"], xlabel=g["xlabel"], ylabel=g["ylabel"])
|
||||||
|
fig.savefig(fig_dir / "cam_perf_multipanel.png", dpi=300, bbox_inches="tight")
|
||||||
|
plt.close(fig)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Orchestrator
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def run_all(
|
||||||
|
configs: list[SweepConfig],
|
||||||
|
output_root: Path = Path("outputs/cam_perf"),
|
||||||
|
date: str | None = None,
|
||||||
|
repo_root: Path | None = None,
|
||||||
|
) -> int:
|
||||||
|
"""Run all sweep configurations sequentially, write CSV, return 0 if all pass.
|
||||||
|
|
||||||
|
Output is written under ``*output_root* / *date*``.
|
||||||
|
"""
|
||||||
|
if repo_root is None:
|
||||||
|
repo_root = Path(__file__).resolve().parent.parent
|
||||||
|
output_root = Path(output_root)
|
||||||
|
if not output_root.is_absolute():
|
||||||
|
output_root = repo_root / output_root
|
||||||
|
if date is None:
|
||||||
|
date = datetime.now().strftime("%Y-%m-%d")
|
||||||
|
output_dir = output_root / date
|
||||||
|
output_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
runs: list[PerfRun] = []
|
||||||
|
for config in configs:
|
||||||
|
run = run_one_config(config, output_dir=output_dir, repo_root=repo_root)
|
||||||
|
runs.append(run)
|
||||||
|
|
||||||
|
write_csv(output_dir, runs)
|
||||||
|
write_summary(output_dir, runs)
|
||||||
|
write_notes(output_dir, runs)
|
||||||
|
plot_figures(output_dir, runs)
|
||||||
|
|
||||||
|
all_pass = all(run.status == "pass" for run in runs)
|
||||||
|
return 0 if all_pass else 1
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Helpers
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def script_repo_root() -> Path:
|
||||||
|
"""Return the repository root (two levels up from ``scripts/sweep_cam_perf.py``)."""
|
||||||
|
return Path(__file__).resolve().parents[1]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# CLI
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
app = typer.Typer(
|
||||||
|
add_completion=False,
|
||||||
|
help=(
|
||||||
|
"Run CAM performance simulation sweep "
|
||||||
|
"and collect paper-ready outputs."
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@app.command()
|
||||||
|
def main(
|
||||||
|
date: Annotated[str | None, typer.Option("--date", help="Output date directory. Defaults to today.")] = None,
|
||||||
|
single: Annotated[bool, typer.Option("--single", help="Run a single config instead of the full sweep.")] = False,
|
||||||
|
num_rows: Annotated[int, typer.Option("--num-rows", help="CAM row count.", min=1)] = 512,
|
||||||
|
hash_bits: Annotated[int, typer.Option("--hash-bits", help="Hash width.", min=1)] = 512,
|
||||||
|
lanes: Annotated[int, typer.Option("--lanes", help="CAM lane count.", min=1)] = 4,
|
||||||
|
) -> None:
|
||||||
|
"""Run the CAM performance simulation sweep."""
|
||||||
|
if single:
|
||||||
|
configs = [SweepConfig("single", num_rows, hash_bits, lanes)]
|
||||||
|
else:
|
||||||
|
configs = default_sweep_configs()
|
||||||
|
|
||||||
|
run_date = date or date_type.today().isoformat()
|
||||||
|
exit_code = run_all(
|
||||||
|
configs=configs,
|
||||||
|
date=run_date,
|
||||||
|
repo_root=script_repo_root(),
|
||||||
|
)
|
||||||
|
raise typer.Exit(exit_code)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
app()
|
||||||
21
tests/conftest.py
Normal file
21
tests/conftest.py
Normal file
@@ -0,0 +1,21 @@
|
|||||||
|
"""pytest configuration — ensure ``scripts/`` is importable via package syntax.
|
||||||
|
|
||||||
|
The ``scripts`` directory is a PEP 420 implicit namespace package (no
|
||||||
|
``__init__.py``). Because ``uv run pytest`` does **not** add the project root
|
||||||
|
to ``sys.path``, ``from scripts.sweep_cam_perf import ...`` would otherwise
|
||||||
|
fail with ``ModuleNotFoundError``.
|
||||||
|
|
||||||
|
This conftest inserts the repository root at the front of ``sys.path`` so that
|
||||||
|
``scripts`` is discoverable as a top-level package.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
_repo_root = Path(__file__).resolve().parent.parent
|
||||||
|
_repo_root_str = str(_repo_root)
|
||||||
|
|
||||||
|
if _repo_root_str not in sys.path:
|
||||||
|
sys.path.insert(0, _repo_root_str)
|
||||||
295
tests/test_sweep_cam_perf.py
Normal file
295
tests/test_sweep_cam_perf.py
Normal file
@@ -0,0 +1,295 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Fast unit tests for scripts/sweep_cam_perf.py (sweep helpers)."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import subprocess
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from scripts.sweep_cam_perf import (
|
||||||
|
CSV_FIELDNAMES,
|
||||||
|
PerfRun,
|
||||||
|
SweepConfig,
|
||||||
|
_run_subprocess,
|
||||||
|
build_make_command,
|
||||||
|
default_sweep_configs,
|
||||||
|
logical_sweeps_for_config,
|
||||||
|
parse_perf_result,
|
||||||
|
plot_figures,
|
||||||
|
run_all,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# parse_perf_result
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_perf_result_extracts_metrics():
|
||||||
|
log = (
|
||||||
|
"PERF_RESULT latency_cycles=148 total_cycles=158 "
|
||||||
|
"accepted_queries=1 completed_queries=1 "
|
||||||
|
"queries_per_cycle=0.006329 status=pass"
|
||||||
|
)
|
||||||
|
result = parse_perf_result(log)
|
||||||
|
assert result == {
|
||||||
|
"latency_cycles": "148",
|
||||||
|
"total_cycles": "158",
|
||||||
|
"accepted_queries": "1",
|
||||||
|
"completed_queries": "1",
|
||||||
|
"queries_per_cycle": "0.006329",
|
||||||
|
"status": "pass",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_perf_result_returns_empty_when_missing():
|
||||||
|
log = "some random simulation output without the marker"
|
||||||
|
assert parse_perf_result(log) == {}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# default_sweep_configs
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_sweep_configs_are_paper_supporting_dense_matrix():
|
||||||
|
configs = default_sweep_configs()
|
||||||
|
assert len(configs) == 19
|
||||||
|
|
||||||
|
tuples = {(c.num_rows, c.hash_bits, c.lanes) for c in configs}
|
||||||
|
|
||||||
|
# NUM_ROWS sweep (lanes fixed at 16)
|
||||||
|
for nr in (64, 128, 192, 256, 384, 512, 768, 1024):
|
||||||
|
assert (nr, 512, 16) in tuples, f"Missing NUM_ROWS config {(nr, 512, 16)}"
|
||||||
|
|
||||||
|
# HASH_BITS sweep (num_rows fixed at 512, lanes fixed at 16)
|
||||||
|
for hb in (64, 128, 192, 256, 320, 384, 448, 512):
|
||||||
|
assert (512, hb, 16) in tuples, f"Missing HASH_BITS config {(512, hb, 16)}"
|
||||||
|
|
||||||
|
# LANES sweep (num_rows and hash_bits fixed at 512)
|
||||||
|
for la in (2, 4, 8, 16, 32):
|
||||||
|
assert (512, 512, la) in tuples, f"Missing LANES config {(512, 512, la)}"
|
||||||
|
|
||||||
|
assert len(tuples) == 19
|
||||||
|
# All configs must satisfy num_rows % lanes == 0
|
||||||
|
for c in configs:
|
||||||
|
assert c.num_rows % c.lanes == 0, (
|
||||||
|
f"num_rows={c.num_rows} not divisible by lanes={c.lanes} for {c}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# logical_sweeps_for_config
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_logical_sweeps_baseline_belongs_to_all_three_groups():
|
||||||
|
"""The baseline (512,512,16) config belongs to all three sweep groups,
|
||||||
|
even though its CSV sweep_name is 'num_rows' (first-occurrence dedup)."""
|
||||||
|
configs = default_sweep_configs()
|
||||||
|
# Locate the shared baseline config.
|
||||||
|
baseline = next(c for c in configs if (c.num_rows, c.hash_bits, c.lanes) == (512, 512, 16))
|
||||||
|
groups = logical_sweeps_for_config(baseline)
|
||||||
|
assert set(groups) == {"num_rows", "hash_bits", "lanes"}, (
|
||||||
|
f"Baseline should belong to all three groups, got {groups}"
|
||||||
|
)
|
||||||
|
assert len(configs) == 19, "Sweep config count must remain 19"
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# build_make_command
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_build_make_command_uses_perf_test_module():
|
||||||
|
cfg = SweepConfig("smoke", 128, 128, 4)
|
||||||
|
cmd = build_make_command(cfg)
|
||||||
|
assert cmd == [
|
||||||
|
"make",
|
||||||
|
"-C",
|
||||||
|
"hw/sim",
|
||||||
|
"NUM_ROWS=128",
|
||||||
|
"HASH_BITS=128",
|
||||||
|
"LANES=4",
|
||||||
|
"COCOTB_TEST_MODULES=tests.test_cam_perf",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# CSV_FIELDNAMES
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_csv_fieldnames_match_spec():
|
||||||
|
assert CSV_FIELDNAMES == [
|
||||||
|
"sweep_name",
|
||||||
|
"num_rows",
|
||||||
|
"hash_bits",
|
||||||
|
"lanes",
|
||||||
|
"latency_cycles",
|
||||||
|
"total_cycles",
|
||||||
|
"accepted_queries",
|
||||||
|
"completed_queries",
|
||||||
|
"queries_per_cycle",
|
||||||
|
"sim_time_sec",
|
||||||
|
"status",
|
||||||
|
"log_path",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# _run_subprocess — timeout preserves partial output
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_subprocess_timeout_preserves_partial_output(monkeypatch):
|
||||||
|
"""Timeout error string includes partial stdout/stderr from the process."""
|
||||||
|
|
||||||
|
def mock_run(*args, **kwargs):
|
||||||
|
raise subprocess.TimeoutExpired(
|
||||||
|
cmd=["mock"],
|
||||||
|
timeout=1,
|
||||||
|
output="partial stdout data",
|
||||||
|
stderr="partial stderr data",
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr(subprocess, "run", mock_run)
|
||||||
|
|
||||||
|
result, duration, error = _run_subprocess(
|
||||||
|
["mock"], cwd=Path("/tmp"), env={}, timeout=1,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is None
|
||||||
|
assert duration >= 0
|
||||||
|
assert "timed out after 1s" in error
|
||||||
|
assert "[partial stdout before timeout]" in error
|
||||||
|
assert "partial stdout data" in error
|
||||||
|
assert "[partial stderr before timeout]" in error
|
||||||
|
assert "partial stderr data" in error
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_subprocess_timeout_without_partial_output(monkeypatch):
|
||||||
|
"""Timeout error is clean when the subprocess produced no output."""
|
||||||
|
|
||||||
|
def mock_run(*args, **kwargs):
|
||||||
|
raise subprocess.TimeoutExpired(cmd=["mock"], timeout=5)
|
||||||
|
|
||||||
|
monkeypatch.setattr(subprocess, "run", mock_run)
|
||||||
|
|
||||||
|
result, duration, error = _run_subprocess(
|
||||||
|
["mock"], cwd=Path("/tmp"), env={}, timeout=5,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is None
|
||||||
|
assert "timed out after 5s" in error
|
||||||
|
assert "[partial stdout" not in error
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# run_all — output_root anchoring
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _make_mock_run_one_config(status="pass"):
|
||||||
|
"""Return a ``run_one_config`` stand-in that never invokes a subprocess."""
|
||||||
|
|
||||||
|
def mock(config, *, output_dir, repo_root, timeout=1800):
|
||||||
|
return PerfRun(
|
||||||
|
config=config,
|
||||||
|
status=status,
|
||||||
|
returncode=0 if status == "pass" else 1,
|
||||||
|
duration_sec=0.0,
|
||||||
|
log_path=Path("logs/test.log"),
|
||||||
|
metrics={"status": status},
|
||||||
|
)
|
||||||
|
|
||||||
|
return mock
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_all_anchors_relative_output_root_to_repo_root(tmp_path, monkeypatch):
|
||||||
|
"""A relative ``output_root`` is resolved under ``repo_root``."""
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"scripts.sweep_cam_perf.run_one_config",
|
||||||
|
_make_mock_run_one_config("pass"),
|
||||||
|
)
|
||||||
|
|
||||||
|
configs = [SweepConfig("test", 512, 512, 16)]
|
||||||
|
exit_code = run_all(
|
||||||
|
configs,
|
||||||
|
output_root="outputs/cam_perf",
|
||||||
|
date="2026-06-01",
|
||||||
|
repo_root=tmp_path,
|
||||||
|
)
|
||||||
|
|
||||||
|
expected_csv = tmp_path / "outputs" / "cam_perf" / "2026-06-01" / "sweep.csv"
|
||||||
|
assert expected_csv.exists(), f"sweep.csv not found at {expected_csv}"
|
||||||
|
assert exit_code == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_run_all_uses_absolute_output_root_unmodified(tmp_path, monkeypatch):
|
||||||
|
"""An absolute ``output_root`` is used as-is, not anchored under repo_root."""
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"scripts.sweep_cam_perf.run_one_config",
|
||||||
|
_make_mock_run_one_config("pass"),
|
||||||
|
)
|
||||||
|
|
||||||
|
configs = [SweepConfig("test", 512, 512, 16)]
|
||||||
|
abs_output = tmp_path / "custom_dir"
|
||||||
|
repo_root = Path("/nonexistent/repo")
|
||||||
|
|
||||||
|
exit_code = run_all(
|
||||||
|
configs,
|
||||||
|
output_root=abs_output,
|
||||||
|
date="2026-06-01",
|
||||||
|
repo_root=repo_root,
|
||||||
|
)
|
||||||
|
|
||||||
|
expected_csv = abs_output / "2026-06-01" / "sweep.csv"
|
||||||
|
assert expected_csv.exists(), f"sweep.csv not found at {expected_csv}"
|
||||||
|
assert exit_code == 0
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# plot_figures — required PNG outputs
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_plot_figures_creates_required_pngs(tmp_path):
|
||||||
|
"""plot_figures creates four PNGs under figures/ when enough pass runs exist
|
||||||
|
across all three logical sweep groups. No exception with Agg backend."""
|
||||||
|
configs = [
|
||||||
|
SweepConfig("num_rows", 128, 512, 16),
|
||||||
|
SweepConfig("num_rows", 256, 512, 16),
|
||||||
|
SweepConfig("num_rows", 512, 512, 16), # baseline: all three groups
|
||||||
|
SweepConfig("hash_bits", 512, 128, 16),
|
||||||
|
SweepConfig("hash_bits", 512, 256, 16),
|
||||||
|
SweepConfig("lanes", 512, 512, 4),
|
||||||
|
SweepConfig("lanes", 512, 512, 8),
|
||||||
|
]
|
||||||
|
runs = [
|
||||||
|
PerfRun(
|
||||||
|
config=cfg,
|
||||||
|
status="pass",
|
||||||
|
returncode=0,
|
||||||
|
duration_sec=1.0,
|
||||||
|
log_path=Path("logs/test.log"),
|
||||||
|
metrics={
|
||||||
|
"latency_cycles": str(50 + cfg.num_rows % 100),
|
||||||
|
"total_cycles": str(100 + cfg.num_rows % 100),
|
||||||
|
"accepted_queries": "10",
|
||||||
|
"completed_queries": "10",
|
||||||
|
"queries_per_cycle": "0.05",
|
||||||
|
"status": "pass",
|
||||||
|
},
|
||||||
|
)
|
||||||
|
for cfg in configs
|
||||||
|
]
|
||||||
|
|
||||||
|
plot_figures(tmp_path, runs)
|
||||||
|
|
||||||
|
fig_dir = tmp_path / "figures"
|
||||||
|
assert fig_dir.is_dir()
|
||||||
|
assert (fig_dir / "cam_num_rows_vs_latency.png").exists()
|
||||||
|
assert (fig_dir / "cam_hash_bits_vs_latency.png").exists()
|
||||||
|
assert (fig_dir / "cam_lanes_vs_throughput.png").exists()
|
||||||
|
assert (fig_dir / "cam_perf_multipanel.png").exists()
|
||||||
Reference in New Issue
Block a user