#!/usr/bin/env python3 """CAM performance simulation sweep — data model, sweep matrix, and command builders.""" from __future__ import annotations import csv import os import re import subprocess import time from dataclasses import dataclass from datetime import date as date_type from datetime import datetime from pathlib import Path from typing import Annotated import typer # Timeout for each external simulation invocation (30 minutes). DEFAULT_COMMAND_TIMEOUT_SEC = 1800 # Cocotb test module that records PERF_RESULT metrics. PERF_MODULE = "tests.test_cam_perf" # CSV column names for sweep result output — must match exactly. CSV_FIELDNAMES = [ "sweep_name", "num_rows", "hash_bits", "lanes", "latency_cycles", "total_cycles", "accepted_queries", "completed_queries", "queries_per_cycle", "sim_time_sec", "status", "log_path", ] # --------------------------------------------------------------------------- # Dense sweep-matrix value sets for paper-quality figures. NUM_ROWS_SWEEP_VALUES = (64, 128, 192, 256, 384, 512, 768, 1024) HASH_BITS_SWEEP_VALUES = (64, 128, 192, 256, 320, 384, 448, 512) LANES_SWEEP_VALUES = (2, 4, 8, 16, 32) # --------------------------------------------------------------------------- # Data model # --------------------------------------------------------------------------- @dataclass(frozen=True) class SweepConfig: """A single point in the CAM performance sweep matrix.""" sweep_name: str num_rows: int hash_bits: int lanes: int @dataclass(frozen=True) class PerfRun: """Result of executing a single sweep configuration.""" config: SweepConfig status: str returncode: int duration_sec: float log_path: Path metrics: dict[str, str] error: str = "" # --------------------------------------------------------------------------- # Default sweep matrix # --------------------------------------------------------------------------- def default_sweep_configs() -> list[SweepConfig]: """Return the paper-supporting dense sweep matrix (19 unique configs). The matrix sweeps one dimension at a time: * NUM_ROWS: ``(nr, 512, 16)`` for ``nr`` in ``NUM_ROWS_SWEEP_VALUES`` * HASH_BITS: ``(512, hb, 16)`` for ``hb`` in ``HASH_BITS_SWEEP_VALUES`` * LANES: ``(512, 512, la)`` for ``la`` in ``LANES_SWEEP_VALUES`` Overlapping triples are deduplicated preserving first occurrence. """ candidates = ( [SweepConfig("num_rows", nr, 512, 16) for nr in NUM_ROWS_SWEEP_VALUES] + [SweepConfig("hash_bits", 512, hb, 16) for hb in HASH_BITS_SWEEP_VALUES] + [SweepConfig("lanes", 512, 512, la) for la in LANES_SWEEP_VALUES] ) seen: set[tuple[int, int, int]] = set() result: list[SweepConfig] = [] for cfg in candidates: key = (cfg.num_rows, cfg.hash_bits, cfg.lanes) if key not in seen: seen.add(key) result.append(cfg) return result # --------------------------------------------------------------------------- # Sweep membership helper # --------------------------------------------------------------------------- def logical_sweeps_for_config(config: SweepConfig) -> tuple[str, ...]: """Return the logical sweep-group memberships for *config*. A single config may participate in multiple sweep dimensions. This is especially important for the baseline config ``(512, 512, 16)`` whose singleton CSV row has ``sweep_name='num_rows'`` (first occurrence wins during dedup), yet logically belongs to **all three** sweep groups. The predicates reference the module-level dense-value constants: * ``'num_rows'`` – ``HASH_BITS=512, LANES=16, NUM_ROWS ∈ NUM_ROWS_SWEEP_VALUES`` * ``'hash_bits'`` – ``NUM_ROWS=512, LANES=16, HASH_BITS ∈ HASH_BITS_SWEEP_VALUES`` * ``'lanes'`` – ``NUM_ROWS=512, HASH_BITS=512, LANES ∈ LANES_SWEEP_VALUES`` Downstream plot / analysis code **must** use this helper to derive sweep groups instead of filtering solely by ``SweepConfig.sweep_name``; otherwise the baseline ``(512,512,16)`` would be missing from ``hash_bits`` and ``lanes`` group queries. """ memberships: list[str] = [] nr, hb, la = config.num_rows, config.hash_bits, config.lanes if hb == 512 and la == 16 and nr in NUM_ROWS_SWEEP_VALUES: memberships.append("num_rows") if nr == 512 and la == 16 and hb in HASH_BITS_SWEEP_VALUES: memberships.append("hash_bits") if nr == 512 and hb == 512 and la in LANES_SWEEP_VALUES: memberships.append("lanes") return tuple(memberships) # --------------------------------------------------------------------------- # Log parsing # --------------------------------------------------------------------------- _PERF_EXPECTED_KEYS = frozenset({ "latency_cycles", "total_cycles", "accepted_queries", "completed_queries", "queries_per_cycle", "status", }) def parse_perf_result(log_text: str) -> dict[str, str]: """Parse the first *complete* ``PERF_RESULT`` marker line from a simulation log. Scans all lines for a ``PERF_RESULT`` marker (standalone word). For each such line, extracts ``key=value`` tokens and checks whether all expected keys (``latency_cycles``, ``total_cycles``, ``accepted_queries``, ``completed_queries``, ``queries_per_cycle``, ``status``) are present. Returns the dict from the **first** line with a complete set, or ``{}`` if no complete marker exists in the entire log. """ _marker_re = re.compile(r"\bPERF_RESULT\b") for line in log_text.splitlines(): if _marker_re.search(line): tokens = line.split() parsed: dict[str, str] = {} for token in tokens: if "=" in token: key, _, value = token.partition("=") parsed[key] = value if _PERF_EXPECTED_KEYS.issubset(parsed.keys()): return parsed return {} # --------------------------------------------------------------------------- # Command builders # --------------------------------------------------------------------------- def build_make_command(config: SweepConfig) -> list[str]: """Build a ``make`` command to run the perf test module with *config*.""" return [ "make", "-C", "hw/sim", f"NUM_ROWS={config.num_rows}", f"HASH_BITS={config.hash_bits}", f"LANES={config.lanes}", f"COCOTB_TEST_MODULES={PERF_MODULE}", ] # --------------------------------------------------------------------------- # Execution helpers # --------------------------------------------------------------------------- def _run_subprocess( cmd: list[str], *, cwd: Path, env: dict[str, str], timeout: int, ) -> tuple[subprocess.CompletedProcess[str] | None, float, str | None]: """Execute *cmd*, capture output, enforce *timeout*. Returns ``(result, duration_sec, error_msg)``. On success *result* is a ``CompletedProcess`` and *error_msg* is ``None``. On failure (timeout / OSError / SubprocessError) *result* is ``None`` and *error_msg* describes what happened. """ start = time.monotonic() try: result = subprocess.run( cmd, cwd=cwd, env=env, text=True, capture_output=True, timeout=timeout, ) except subprocess.TimeoutExpired as exc: end = time.monotonic() partial = "" if exc.stdout: partial += f"\n# [partial stdout before timeout]\n{exc.stdout}" if exc.stderr: partial += f"\n# [partial stderr before timeout]\n{exc.stderr}" return None, round(end - start, 3), f"timed out after {timeout}s{partial}" except (OSError, subprocess.SubprocessError) as exc: end = time.monotonic() return None, round(end - start, 3), str(exc) end = time.monotonic() return result, round(end - start, 3), None def run_one_config( config: SweepConfig, *, output_dir: Path, repo_root: Path, timeout: int = DEFAULT_COMMAND_TIMEOUT_SEC, ) -> PerfRun: """Execute a single sweep configuration and return the result. Steps: 1. Create logs directory. 2. Validate ``NUM_ROWS % LANES == 0``; return fail immediately if invalid. 3. Run ``make -C hw/sim clean``. 4. Build and run the simulation command via ``build_make_command(config)``. 5. Capture combined stdout/stderr into a log file named ``_r_b_l.log``. 6. Parse ``PERF_RESULT`` from the combined log. 7. Determine pass/fail: pass only if returncode == 0 and metrics status == 'pass'. """ logs_dir = output_dir / "logs" logs_dir.mkdir(parents=True, exist_ok=True) log_filename = ( f"{config.sweep_name}_{config.num_rows}r_{config.hash_bits}b_{config.lanes}l.log" ) log_path = logs_dir / log_filename relative_log = Path("logs") / log_filename # ── Validation: NUM_ROWS must be divisible by LANES ────────────────── if config.num_rows % config.lanes != 0: log_path.write_text( f"# ERROR: NUM_ROWS ({config.num_rows}) not divisible by LANES ({config.lanes})\n" ) return PerfRun( config=config, status="fail", returncode=-1, duration_sec=0.0, log_path=relative_log, metrics={}, error=f"NUM_ROWS ({config.num_rows}) not divisible by LANES ({config.lanes})", ) env = os.environ.copy() log_chunks: list[str] = [] # ── Clean step ─────────────────────────────────────────────────────── clean_cmd = ["make", "-C", "hw/sim", "clean"] log_chunks.append(f"# Command: {' '.join(clean_cmd)}\n") clean_result, clean_duration, clean_err = _run_subprocess( clean_cmd, cwd=repo_root, env=env, timeout=timeout, ) if clean_result is not None: log_chunks.append(clean_result.stdout) log_chunks.append(clean_result.stderr) else: log_chunks.append(f"# ERROR: {clean_err}\n") if clean_err is not None or ( clean_result is not None and clean_result.returncode != 0 ): returncode = clean_result.returncode if clean_result is not None else -1 log_path.write_text("".join(log_chunks)) return PerfRun( config=config, status="fail", returncode=returncode, duration_sec=clean_duration, log_path=relative_log, metrics={}, error=clean_err or f"make clean failed with returncode {returncode}", ) # ── Main simulation command ────────────────────────────────────────── command = build_make_command(config) log_chunks.append(f"# Command: {' '.join(command)}\n") main_result, duration, main_err = _run_subprocess( command, cwd=repo_root, env=env, timeout=timeout, ) if main_result is not None: log_chunks.append(main_result.stdout) log_chunks.append(main_result.stderr) else: log_chunks.append(f"# ERROR: {main_err}\n") combined_log = "".join(log_chunks) log_path.write_text(combined_log) # ── Parse PERF_RESULT ──────────────────────────────────────────────── metrics = parse_perf_result(combined_log) if main_err is not None: status = "fail" returncode = -1 error = main_err else: returncode = main_result.returncode metrics_status = metrics.get("status", "") if returncode == 0 and metrics_status == "pass": status = "pass" error = "" else: status = "fail" if returncode != 0: error = f"simulation returned non-zero exit code {returncode}" elif metrics_status != "pass": error = f"PERF_RESULT status is '{metrics_status}'" else: error = "unknown failure" return PerfRun( config=config, status=status, returncode=returncode, duration_sec=duration, log_path=relative_log, metrics=metrics, error=error, ) # --------------------------------------------------------------------------- # CSV helpers # --------------------------------------------------------------------------- def run_to_row(run: PerfRun) -> dict[str, str]: """Convert a ``PerfRun`` to a CSV row dict matching ``CSV_FIELDNAMES``.""" return { "sweep_name": run.config.sweep_name, "num_rows": str(run.config.num_rows), "hash_bits": str(run.config.hash_bits), "lanes": str(run.config.lanes), "latency_cycles": run.metrics.get("latency_cycles", ""), "total_cycles": run.metrics.get("total_cycles", ""), "accepted_queries": run.metrics.get("accepted_queries", ""), "completed_queries": run.metrics.get("completed_queries", ""), "queries_per_cycle": run.metrics.get("queries_per_cycle", ""), "sim_time_sec": str(run.duration_sec), "status": run.status, "log_path": str(run.log_path), } def write_csv(output_dir: Path, runs: list[PerfRun]) -> None: """Write ``sweep.csv`` with ``CSV_FIELDNAMES`` header and one row per run.""" path = output_dir / "sweep.csv" with open(path, "w", newline="") as f: writer = csv.DictWriter(f, fieldnames=CSV_FIELDNAMES) writer.writeheader() for run in runs: writer.writerow(run_to_row(run)) # --------------------------------------------------------------------------- # Output helpers # --------------------------------------------------------------------------- def _float_or_none(value: str) -> float | None: """Parse *value* as ``float``, returning ``None`` on failure.""" try: return float(value) except (ValueError, TypeError): return None # --------------------------------------------------------------------------- # Nature-like plot style constants # --------------------------------------------------------------------------- PLOT_NAVY = "#243B53" PLOT_SLATE = "#7A8793" PLOT_TEAL = "#4FA7A3" PLOT_GRID = "#D7DEE5" PLOT_TEXT = "#1F2933" def _style_perf_axis(ax, *, title: str, xlabel: str, ylabel: str) -> None: """Apply Nature-like styling to a performance figure axis.""" ax.set_title(title, fontsize=10, color=PLOT_TEXT, pad=8) ax.set_xlabel(xlabel, fontsize=9, color=PLOT_TEXT) ax.set_ylabel(ylabel, fontsize=9, color=PLOT_TEXT) ax.grid(True, color=PLOT_GRID, linewidth=0.6, alpha=0.8) ax.spines["top"].set_visible(False) ax.spines["right"].set_visible(False) ax.spines["left"].set_color(PLOT_SLATE) ax.spines["bottom"].set_color(PLOT_SLATE) ax.tick_params(axis="both", colors=PLOT_TEXT, labelsize=8, width=0.6) def _plot_points( runs: list[PerfRun], group: str, x_getter, metric: str, ) -> list[tuple[int, float]]: """Filter *runs* to pass runs in *group* and return sorted ``(x, y)`` points.""" points: list[tuple[int, float]] = [] for run in runs: if run.status != "pass" or group not in logical_sweeps_for_config(run.config): continue value = _float_or_none(run.metrics.get(metric, "")) if value is None: continue points.append((int(x_getter(run.config)), value)) return sorted(points, key=lambda item: item[0]) def write_summary(output_dir: Path, runs: list[PerfRun]) -> None: """Write ``sweep_summary.md`` in Chinese with summary table and disclaimer.""" total = len(runs) passed = sum(1 for r in runs if r.status == "pass") failed = total - passed failed_runs = [r for r in runs if r.status == "fail"] lines: list[str] = [] lines.append("# CAM 性能扫描摘要") lines.append("") lines.append(f"**输出目录:** {output_dir}") lines.append(f"**总运行数:** {total}") lines.append(f"**通过数:** {passed}") lines.append(f"**失败数:** {failed}") lines.append("") lines.append("## 性能结果") lines.append("") lines.append("| 扫描分组 | NUM_ROWS | HASH_BITS | LANES | 延迟(周期数) | 查询量/周期 | 状态 | 日志 |") lines.append("|----------|----------|-----------|-------|-------------|-------------|------|------|") for r in runs: lines.append( f"| {r.config.sweep_name} | {r.config.num_rows} | {r.config.hash_bits} " f"| {r.config.lanes} | {r.metrics.get('latency_cycles', '')} " f"| {r.metrics.get('queries_per_cycle', '')} | {r.status} " f"| {r.log_path} |" ) lines.append("") lines.append("## 失败列表") lines.append("") if failed_runs: for r in failed_runs: err = r.error or "无" lines.append(f"- **{r.config.sweep_name}** (`{r.log_path}`): {err}") else: lines.append("- 无") lines.append("") lines.append("## 免责声明") lines.append("") lines.append( "本报告为 **Verilator/Cocotb 仿真** 结果,非 FPGA 板级实测性能、" "资源利用率或 Fmax。所有数据仅供架构评估参考,不代表最终 FPGA 实现指标。" ) lines.append("") path = output_dir / "sweep_summary.md" path.write_text("\n".join(lines)) def write_notes(output_dir: Path, runs: list[PerfRun]) -> None: """Write ``ch6_3_cam_perf_notes.md`` in Chinese with trend analysis.""" lines: list[str] = [] lines.append("# 第 6.3 节 CAM 性能仿真笔记") lines.append("") lines.append("## 说明") lines.append("") lines.append( "以下结果基于 **Verilator/Cocotb 仿真** 环境,旨在评估 CAM 设计的查询延迟" "(latency_cycles)和吞吐量(queries_per_cycle)趋势。所有数据均为仿真" "结果,**非 FPGA 板级实测性能**。本文档不提供 LUT、FF、BRAM 或 Fmax 等" "资源利用率或时序结论。" ) lines.append("") lines.append("## 参数趋势分析") lines.append("") lines.append("### NUM_ROWS(CAM 行数)") lines.append("") nr_runs = [ r for r in runs if "num_rows" in logical_sweeps_for_config(r.config) and r.status == "pass" ] if nr_runs: for r in sorted(nr_runs, key=lambda x: x.config.num_rows): lat = r.metrics.get("latency_cycles", "N/A") qpc = r.metrics.get("queries_per_cycle", "N/A") lines.append(f"- NUM_ROWS={r.config.num_rows}: 延迟={lat} 周期, 吞吐量={qpc} 查询/周期") else: lines.append("- 无通过数据") lines.append("") lines.append("### HASH_BITS(哈希位宽)") lines.append("") hb_runs = [ r for r in runs if "hash_bits" in logical_sweeps_for_config(r.config) and r.status == "pass" ] if hb_runs: for r in sorted(hb_runs, key=lambda x: x.config.hash_bits): lat = r.metrics.get("latency_cycles", "N/A") qpc = r.metrics.get("queries_per_cycle", "N/A") lines.append(f"- HASH_BITS={r.config.hash_bits}: 延迟={lat} 周期, 吞吐量={qpc} 查询/周期") else: lines.append("- 无通过数据") lines.append("") lines.append("### LANES(流水线通路数)") lines.append("") la_runs = [ r for r in runs if "lanes" in logical_sweeps_for_config(r.config) and r.status == "pass" ] if la_runs: for r in sorted(la_runs, key=lambda x: x.config.lanes): lat = r.metrics.get("latency_cycles", "N/A") qpc = r.metrics.get("queries_per_cycle", "N/A") lines.append(f"- LANES={r.config.lanes}: 延迟={lat} 周期, 吞吐量={qpc} 查询/周期") else: lines.append("- 无通过数据") lines.append("") lines.append("## 免责声明") lines.append("") lines.append( "本文档中的数据来源于 Verilator/Cocotb 仿真,用于观察性能趋势。" "实际 FPGA 实现的性能、资源利用率和最大工作频率(Fmax)可能因综合选项、" "布局布线、芯片型号等因素而显著不同。请勿将本文档中的数据直接用作 FPGA " "实现指标。" ) lines.append("") path = output_dir / "ch6_3_cam_perf_notes.md" path.write_text("\n".join(lines)) def plot_figures(output_dir: Path, runs: list[PerfRun]) -> None: """Generate performance trend figures under ``figures/``. Three single-panel PNGs and one 1×3 multi-panel PNG are produced when at least 2 pass rows exist in a logical sweep group: - ``cam_num_rows_vs_latency.png`` - ``cam_hash_bits_vs_latency.png`` - ``cam_lanes_vs_throughput.png`` - ``cam_perf_multipanel.png`` Uses ``logical_sweeps_for_config`` so the baseline ``(512,512,16)`` appears in all three groups. """ pass_runs = [r for r in runs if r.status == "pass"] if not pass_runs: return import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt # White / Nature-like rcParams defaults. plt.rcParams.update({ "figure.facecolor": "white", "axes.facecolor": "white", "savefig.facecolor": "white", "font.family": "DejaVu Sans", "axes.linewidth": 0.8, }) fig_dir = output_dir / "figures" fig_dir.mkdir(parents=True, exist_ok=True) # ── Define the three logical groups ────────────────────────────────── groups: list[dict] = [ { "group": "num_rows", "x_getter": lambda c: c.num_rows, "metric": "latency_cycles", "xlabel": "NUM_ROWS", "ylabel": "Latency (cycles)", "title": "CAM Query Latency vs NUM_ROWS\n(HASH_BITS=512, LANES=16)", "filename": "cam_num_rows_vs_latency.png", "marker": "o", }, { "group": "hash_bits", "x_getter": lambda c: c.hash_bits, "metric": "latency_cycles", "xlabel": "HASH_BITS", "ylabel": "Latency (cycles)", "title": "CAM Query Latency vs HASH_BITS\n(NUM_ROWS=512, LANES=16)", "filename": "cam_hash_bits_vs_latency.png", "marker": "s", }, { "group": "lanes", "x_getter": lambda c: c.lanes, "metric": "queries_per_cycle", "xlabel": "LANES", "ylabel": "Queries / Cycle", "title": "CAM Throughput vs LANES\n(NUM_ROWS=512, HASH_BITS=512)", "filename": "cam_lanes_vs_throughput.png", "marker": "^", }, ] # ── Individual figures ─────────────────────────────────────────────── for g in groups: points = _plot_points(pass_runs, g["group"], g["x_getter"], g["metric"]) if len(points) < 2: continue xs, ys = zip(*points) fig, ax = plt.subplots(figsize=(4.2, 3.0)) ax.plot( xs, ys, color=PLOT_NAVY, linewidth=1.6, marker=g["marker"], markersize=4.5, markerfacecolor=PLOT_TEAL, markeredgecolor="white", markeredgewidth=0.7, ) _style_perf_axis(ax, title=g["title"], xlabel=g["xlabel"], ylabel=g["ylabel"]) fig.tight_layout(pad=0.7) fig.savefig(fig_dir / g["filename"], dpi=300, bbox_inches="tight") plt.close(fig) # ── Multi-panel figure (1 × 3) ─────────────────────────────────────── fig, axes = plt.subplots(1, 3, figsize=(10.8, 3.0), constrained_layout=True) for ax, g in zip(axes, groups): points = _plot_points(pass_runs, g["group"], g["x_getter"], g["metric"]) if len(points) < 2: ax.text(0.5, 0.5, "insufficient data", ha="center", va="center", transform=ax.transAxes, fontsize=8, color=PLOT_SLATE) continue xs, ys = zip(*points) ax.plot( xs, ys, color=PLOT_NAVY, linewidth=1.6, marker=g["marker"], markersize=4.5, markerfacecolor=PLOT_TEAL, markeredgecolor="white", markeredgewidth=0.7, ) _style_perf_axis(ax, title=g["title"], xlabel=g["xlabel"], ylabel=g["ylabel"]) fig.savefig(fig_dir / "cam_perf_multipanel.png", dpi=300, bbox_inches="tight") plt.close(fig) # --------------------------------------------------------------------------- # Orchestrator # --------------------------------------------------------------------------- def run_all( configs: list[SweepConfig], output_root: Path = Path("outputs/cam_perf"), date: str | None = None, repo_root: Path | None = None, ) -> int: """Run all sweep configurations sequentially, write CSV, return 0 if all pass. Output is written under ``*output_root* / *date*``. """ if repo_root is None: repo_root = Path(__file__).resolve().parent.parent output_root = Path(output_root) if not output_root.is_absolute(): output_root = repo_root / output_root if date is None: date = datetime.now().strftime("%Y-%m-%d") output_dir = output_root / date output_dir.mkdir(parents=True, exist_ok=True) runs: list[PerfRun] = [] for config in configs: run = run_one_config(config, output_dir=output_dir, repo_root=repo_root) runs.append(run) write_csv(output_dir, runs) write_summary(output_dir, runs) write_notes(output_dir, runs) plot_figures(output_dir, runs) all_pass = all(run.status == "pass" for run in runs) return 0 if all_pass else 1 # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- def script_repo_root() -> Path: """Return the repository root (two levels up from ``scripts/sweep_cam_perf.py``).""" return Path(__file__).resolve().parents[1] # --------------------------------------------------------------------------- # CLI # --------------------------------------------------------------------------- app = typer.Typer( add_completion=False, help=( "Run CAM performance simulation sweep " "and collect paper-ready outputs." ), ) @app.command() def main( date: Annotated[str | None, typer.Option("--date", help="Output date directory. Defaults to today.")] = None, single: Annotated[bool, typer.Option("--single", help="Run a single config instead of the full sweep.")] = False, num_rows: Annotated[int, typer.Option("--num-rows", help="CAM row count.", min=1)] = 512, hash_bits: Annotated[int, typer.Option("--hash-bits", help="Hash width.", min=1)] = 512, lanes: Annotated[int, typer.Option("--lanes", help="CAM lane count.", min=1)] = 4, ) -> None: """Run the CAM performance simulation sweep.""" if single: configs = [SweepConfig("single", num_rows, hash_bits, lanes)] else: configs = default_sweep_configs() run_date = date or date_type.today().isoformat() exit_code = run_all( configs=configs, date=run_date, repo_root=script_repo_root(), ) raise typer.Exit(exit_code) if __name__ == "__main__": app()