feat(hw/rtl): implement full Top-K CAM search pipeline with serial result output

- add TOPK_K, FIFO_DEPTH, RESULT_SERIAL parameters to cam_params
- add candidate_fifo: synchronous ready/valid FIFO for (row, score) candidates
- add topk_tracker: tracks top-K candidates with clear/ready handshake
- add result_serializer: serializes packed Top-K array into rank-ordered stream
- refactor match_engine_pipeline from 4-state to 8-state Top-K pipeline
- extend cam_top with serial Top-K interface (result_rank/row/score/last)
- add backward-compatible top1_index/top1_score aliases from rank-0 beat
- add comprehensive tests for all new modules
This commit is contained in:
2026-05-19 18:19:05 +08:00
parent 8bcad1f23f
commit e4cbb5e30d
21 changed files with 2152 additions and 124 deletions

View File

@@ -6,4 +6,24 @@ TOPLEVEL := match_engine_pipeline
COCOTB_TEST_MODULES := tests.modules.match_engine_pipeline.test_match_engine_pipeline
VERILOG_SOURCES := $(RTL_MATCH_ENGINE)
# Allow override from command line (e.g. make TOPK_K=1)
TOPK_K ?= 4
EXTRA_ARGS += +define+TOPK_K=$(TOPK_K)
export TOPK_K
# Parameter-specific build directory so K variants never collide.
SIM_BUILD := sim_build/k_$(TOPK_K)
include $(SIM_ROOT)/mk/cocotb-common.mk
# ── K-specific test targets ──────────────────────────────────────────────
# Recompile with K=1 and filter to the K=1 test.
.PHONY: test-k1
test-k1:
$(MAKE) -B -f Makefile results.xml TOPK_K=1 \
COCOTB_TEST_FILTER=match_engine_topk_k1
# Also remove all module-local build artifacts, including legacy and variant dirs.
clean::
rm -rf sim_build

View File

@@ -1,8 +1,13 @@
from __future__ import annotations
import os
import cocotb
from cocotb.clock import Clock
from cocotb.triggers import RisingEdge
from cocotb.triggers import RisingEdge, Timer
# Read TOPK_K from environment (set by Makefile); default 4 for K=4-only tests.
TOPK_K = int(os.environ.get("TOPK_K", "4"))
async def reset_match(dut):
@@ -21,8 +26,99 @@ async def reset_match(dut):
await RisingEdge(dut.clk)
def score_for(query: int, value: int, width: int) -> int:
"""Compute popcount of ~(query ^ value), i.e. number of matching bits."""
return width - ((query ^ value) & ((1 << width) - 1)).bit_count()
def golden_topk(
rows_and_hashes: list[tuple[int, int]], query: int, width: int, k: int
) -> list[tuple[int, int]]:
"""Return the Top-K (row, score) pairs sorted by score desc, row asc."""
scored = [(row, score_for(query, value, width))
for row, value in rows_and_hashes]
return sorted(scored, key=lambda item: (-item[1], item[0]))[:k]
async def collect_serial_results(
dut, k: int, timeout_cycles: int = 200
) -> list[tuple[int, int, int, int]]:
"""Collect serialised Top-K results from the pipeline.
Returns list of (rank, row, score, last) tuples.
Asserts timeout if the stream does not complete.
"""
dut.result_ready.value = 1
observed: list[tuple[int, int, int, int]] = []
for _ in range(timeout_cycles):
if int(dut.result_valid.value):
rank = int(dut.result_rank.value) if hasattr(
dut, "result_rank") else len(observed)
row = int(dut.result_row.value)
score = int(dut.result_score.value)
last = int(dut.result_last.value) if hasattr(
dut, "result_last") else int(len(observed) == k - 1)
observed.append((rank, row, score, last))
if last:
return observed
await RisingEdge(dut.clk)
raise AssertionError(
f"result stream did not complete after {timeout_cycles} cycles"
)
async def feed_all_batches(
dut,
rows_and_hashes: list[tuple[int, int]] | None = None,
lane_mask: int | None = None,
):
"""Feed all batches to the pipeline.
If *rows_and_hashes* is provided, the test uses those values for each row
(row_id -> hash) instead of the default hash=query (with special rows).
If *lane_mask* is provided, only those lanes are marked valid in each batch.
"""
LANES = len(dut.rd_lane_valid_i)
ROW_BITS = len(dut.rd_row_ids_i) // LANES
HASH_BITS = len(dut.rd_hashes_i) // LANES
NUM_ROWS = 1 << len(dut.rd_base_row_o)
for base in range(0, NUM_ROWS, LANES):
while int(dut.rd_valid_o.value) == 0:
await RisingEdge(dut.clk)
assert int(dut.rd_base_row_o.value) == base
rows = 0
hashes = 0
for lane in range(LANES):
row_id = base + lane
rows |= row_id << (lane * ROW_BITS)
if rows_and_hashes is not None and row_id in rows_and_hashes:
row_hash = rows_and_hashes[row_id]
else:
row_hash = 0
hashes |= row_hash << (lane * HASH_BITS)
dut.rd_row_ids_i.value = rows
dut.rd_hashes_i.value = hashes
if lane_mask is not None:
dut.rd_lane_valid_i.value = lane_mask
else:
dut.rd_lane_valid_i.value = (1 << LANES) - 1
dut.rd_valid_i.value = 1
await RisingEdge(dut.clk)
dut.rd_valid_i.value = 0
# ==============================================================================
# Test: Top-1 smoke
# ==============================================================================
@cocotb.test()
async def match_engine_returns_top1_after_pipeline_drain(dut):
"""Top-1 smoke test — verify basic result_valid/result_row/result_score.
Works with both old Top-1 and new Top-K serial interfaces.
"""
cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start())
await reset_match(dut)
@@ -38,23 +134,8 @@ async def match_engine_returns_top1_after_pipeline_drain(dut):
await RisingEdge(dut.clk)
dut.query_valid.value = 0
for base in range(0, NUM_ROWS, LANES):
while int(dut.rd_valid_o.value) == 0:
await RisingEdge(dut.clk)
assert int(dut.rd_base_row_o.value) == base
rows = 0
hashes = 0
for lane in range(LANES):
row_id = base + lane
rows |= row_id << (lane * ROW_BITS)
row_hash = query if row_id == TARGET_ROW else 0
hashes |= row_hash << (lane * HASH_BITS)
dut.rd_row_ids_i.value = rows
dut.rd_hashes_i.value = hashes
dut.rd_lane_valid_i.value = (1 << LANES) - 1
dut.rd_valid_i.value = 1
await RisingEdge(dut.clk)
dut.rd_valid_i.value = 0
rows_and_hashes: dict[int, int] = {9: query}
await feed_all_batches(dut, rows_and_hashes)
for _ in range(20):
if int(dut.result_valid.value):
@@ -64,3 +145,417 @@ async def match_engine_returns_top1_after_pipeline_drain(dut):
assert int(dut.result_valid.value) == 1
assert int(dut.result_row.value) == TARGET_ROW
assert int(dut.result_score.value) == HASH_BITS
# ==============================================================================
# Test: Top-K all-valid
# ==============================================================================
@cocotb.test()
async def match_engine_keeps_multiple_topk_candidates_from_same_batch(dut):
"""Verify Top-K pipeline returns K best matches in rank order.
Feed all rows with known hash values so the golden Top-K is predictable.
"""
if TOPK_K != 4:
dut._log.info(f"Skipping: hardcoded K=4 test (TOPK_K={TOPK_K})")
return
cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start())
await reset_match(dut)
LANES = len(dut.rd_lane_valid_i)
ROW_BITS = len(dut.rd_row_ids_i) // LANES
HASH_BITS = len(dut.rd_hashes_i) // LANES
NUM_ROWS = 1 << len(dut.rd_base_row_o)
K = 4
query = (1 << HASH_BITS) - 1
special = {0: query, 1: query ^ 1, 2: query ^ 3, 3: query ^ 7}
rows_and_hashes: list[tuple[int, int]] = []
dut.query_hash.value = query
dut.query_valid.value = 1
await RisingEdge(dut.clk)
dut.query_valid.value = 0
for base in range(0, NUM_ROWS, LANES):
while int(dut.rd_valid_o.value) == 0:
await RisingEdge(dut.clk)
rows = 0
hashes = 0
for lane in range(LANES):
row_id = base + lane
row_hash = special.get(row_id, 0)
rows_and_hashes.append((row_id, row_hash))
rows |= row_id << (lane * ROW_BITS)
hashes |= row_hash << (lane * HASH_BITS)
dut.rd_row_ids_i.value = rows
dut.rd_hashes_i.value = hashes
dut.rd_lane_valid_i.value = (1 << LANES) - 1
dut.rd_valid_i.value = 1
await RisingEdge(dut.clk)
dut.rd_valid_i.value = 0
expected = golden_topk(rows_and_hashes, query, HASH_BITS, K)
observed = await collect_serial_results(dut, K, timeout_cycles=800)
assert [(row, score) for _, row, score, _ in observed] == expected, (
f"observed={[(r, s) for _, r, s, _ in observed]} != expected={expected}"
)
assert [rank for rank, _, _, _ in observed] == list(range(K)), (
"ranks are not sequential"
)
assert observed[-1][3] == 1, "last flag not set on final result"
# ==============================================================================
# Test: Invalid-mask coverage — lane 0 invalid, all-invalid batch
# ==============================================================================
@cocotb.test()
async def match_engine_handles_invalid_lane_masks(dut):
"""Pipeline completes correctly with invalid lanes and all-invalid batches.
Coverage:
- Batch 0 is driven with rd_lane_valid_i=0 (all-invalid)
→ batch_scores_done vacuously true, zero candidates pushed.
- All other batches use lane_mask = 0b0110 (lanes 1 and 2 only).
- Invalid lanes (0, 3-7) carry high-score trap hashes that WOULD enter
the Top-K if the pipeline wrongly pushed them.
- Golden Top-K is computed from valid-lane rows only and compared
against the full serialised output — not just a subset check.
"""
if TOPK_K != 4:
dut._log.info(f"Skipping: hardcoded K=4 test (TOPK_K={TOPK_K})")
return
cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start())
await reset_match(dut)
LANES = len(dut.rd_lane_valid_i)
ROW_BITS = len(dut.rd_row_ids_i) // LANES
HASH_BITS = len(dut.rd_hashes_i) // LANES
NUM_ROWS = 1 << len(dut.rd_base_row_o)
K = 4
query = (1 << HASH_BITS) - 1
trap_hash = query # perfect score; would be Top-1 if wrongly pushed
# Only rows on valid lanes (1, 2) from batches AFTER the all-invalid one
# contribute candidates. Use distinct scores so the golden ordering
# is unambiguous.
valid_rows: dict[int, int] = {9: query, 10: query ^ 1}
# Track hash values passed per row (including trap rows on invalid lanes)
# for golden-topk computation.
valid_rows_and_hashes: list[tuple[int, int]] = []
dut.query_hash.value = query
dut.query_valid.value = 1
await RisingEdge(dut.clk)
dut.query_valid.value = 0
for base in range(0, NUM_ROWS, LANES):
while int(dut.rd_valid_o.value) == 0:
await RisingEdge(dut.clk)
# Batch 0 is all-invalid; all others use lanes 1,2 only.
cur_mask = 0x00 if base == 0 else 0b0110
rows = 0
hashes = 0
for lane in range(LANES):
row_id = base + lane
rows |= row_id << (lane * ROW_BITS)
# On invalid lanes drive a trap hash (perfect score).
# On valid lanes drive the designated hash (or 0 if not special).
if cur_mask & (1 << lane):
row_hash = valid_rows.get(row_id, 0)
valid_rows_and_hashes.append((row_id, row_hash))
else:
row_hash = trap_hash # would be Top-1 if wrongly pushed
hashes |= row_hash << (lane * HASH_BITS)
dut.rd_row_ids_i.value = rows
dut.rd_hashes_i.value = hashes
dut.rd_lane_valid_i.value = cur_mask
dut.rd_valid_i.value = 1
await RisingEdge(dut.clk)
dut.rd_valid_i.value = 0
expected = golden_topk(valid_rows_and_hashes, query, HASH_BITS, K)
observed = await collect_serial_results(dut, K, timeout_cycles=800)
assert [(row, score) for _, row, score, _ in observed] == expected, (
f"observed={[(r, s) for _, r, s, _ in observed]} != expected={expected}"
)
assert [rank for rank, _, _, _ in observed] == list(range(K)), (
"ranks are not sequential"
)
assert observed[-1][3] == 1, "last flag not set on final result"
# ==============================================================================
# Test: result_ready stall stability
# ==============================================================================
@cocotb.test()
async def match_engine_result_ready_stall_preserves_output(dut):
"""When result_ready is deasserted while result_valid is high, the output
signals (result_rank, result_row, result_score, result_last) must remain
stable until result_ready is re-asserted.
"""
if TOPK_K != 4:
dut._log.info(f"Skipping: hardcoded K=4 test (TOPK_K={TOPK_K})")
return
cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start())
await reset_match(dut)
LANES = len(dut.rd_lane_valid_i)
ROW_BITS = len(dut.rd_row_ids_i) // LANES
HASH_BITS = len(dut.rd_hashes_i) // LANES
NUM_ROWS = 1 << len(dut.rd_base_row_o)
K = 4
query = (1 << HASH_BITS) - 1
special = {0: query, 1: query ^ 1, 2: query ^ 3, 3: query ^ 7}
rows_and_hashes: list[tuple[int, int]] = []
dut.query_hash.value = query
dut.query_valid.value = 1
await RisingEdge(dut.clk)
dut.query_valid.value = 0
await feed_all_batches(dut)
# Wait for first result
for _ in range(200):
await RisingEdge(dut.clk)
if int(dut.result_valid.value):
break
assert int(dut.result_valid.value) == 1, "result_valid never went high"
# Capture the first-beat values
rank0 = int(dut.result_rank.value)
row0 = int(dut.result_row.value)
score0 = int(dut.result_score.value)
last0 = int(dut.result_last.value)
# Stall — deassert result_ready for several cycles
dut.result_ready.value = 0
for _ in range(5):
await RisingEdge(dut.clk)
assert int(dut.result_valid.value) == 1, (
"result_valid dropped during stall"
)
assert int(dut.result_rank.value) == rank0
assert int(dut.result_row.value) == row0
assert int(dut.result_score.value) == score0
assert int(dut.result_last.value) == last0
# Release stall — should advance to next rank
dut.result_ready.value = 1
await RisingEdge(dut.clk)
# Collect the rest
observed: list[tuple[int, int, int, int]] = [(rank0, row0, score0, last0)]
for _ in range(200):
if int(dut.result_valid.value):
rank = int(dut.result_rank.value)
row = int(dut.result_row.value)
score = int(dut.result_score.value)
last = int(dut.result_last.value)
observed.append((rank, row, score, last))
if last:
break
await RisingEdge(dut.clk)
assert len(observed) == K, (
f"expected {K} results, got {len(observed)}"
)
assert observed[-1][3] == 1, "last flag not set on final result"
# ==============================================================================
# Test: Toggled ready with golden comparison
# ==============================================================================
@cocotb.test()
async def match_engine_holds_serial_result_under_ready_stalls(dut):
"""Toggle result_ready during serialisation and verify golden Top-K matches.
Uses query=(1<<HASH_BITS)-1 and row_hash=query^row_id for deterministic
per-row scoring. result_ready is deasserted every 3rd cycle to exercise
backpressure. Observed beats are captured only on valid+ready handshakes
and compared against golden_topk.
"""
if TOPK_K != 4:
dut._log.info(f"Skipping: hardcoded K=4 test (TOPK_K={TOPK_K})")
return
cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start())
await reset_match(dut)
LANES = len(dut.rd_lane_valid_i)
ROW_BITS = len(dut.rd_row_ids_i) // LANES
HASH_BITS = len(dut.rd_hashes_i) // LANES
NUM_ROWS = 1 << len(dut.rd_base_row_o)
K = TOPK_K
query = (1 << HASH_BITS) - 1
dut.query_hash.value = query
dut.query_valid.value = 1
await RisingEdge(dut.clk)
dut.query_valid.value = 0
# Feed all rows with deterministic hash = query ^ row_id
rows_and_hashes: list[tuple[int, int]] = []
for base in range(0, NUM_ROWS, LANES):
while int(dut.rd_valid_o.value) == 0:
await RisingEdge(dut.clk)
rows = 0
hashes = 0
for lane in range(LANES):
row_id = base + lane
row_hash = query ^ row_id
rows_and_hashes.append((row_id, row_hash))
rows |= row_id << (lane * ROW_BITS)
hashes |= row_hash << (lane * HASH_BITS)
dut.rd_row_ids_i.value = rows
dut.rd_hashes_i.value = hashes
dut.rd_lane_valid_i.value = (1 << LANES) - 1
dut.rd_valid_i.value = 1
await RisingEdge(dut.clk)
dut.rd_valid_i.value = 0
expected = golden_topk(rows_and_hashes, query, HASH_BITS, K)
# Collect with toggling result_ready: stall on every 3rd cycle (cycle % 3 == 1)
observed: list[tuple[int, int]] = []
for cycle in range(800):
dut.result_ready.value = 0 if cycle % 3 == 1 else 1
await RisingEdge(dut.clk)
if int(dut.result_valid.value) and int(dut.result_ready.value):
row = int(dut.result_row.value)
score = int(dut.result_score.value)
observed.append((row, score))
if int(dut.result_last.value):
break
assert observed == expected, (
f"observed={observed} != expected={expected}"
)
# ==============================================================================
# Test: consumed_candidates_debug (SIM_DEBUG only)
# ==============================================================================
@cocotb.test()
async def match_engine_debug_consumed_candidates(dut):
"""Under SIM_DEBUG, consumed_candidates_debug must equal NUM_ROWS after
an all-lane-valid query completes (every row produces one candidate).
"""
if not hasattr(dut, "consumed_candidates_debug"):
dut._log.info("Skipping: consumed_candidates_debug not present (no SIM_DEBUG)")
return
cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start())
await reset_match(dut)
LANES = len(dut.rd_lane_valid_i)
ROW_BITS = len(dut.rd_row_ids_i) // LANES
HASH_BITS = len(dut.rd_hashes_i) // LANES
NUM_ROWS = 1 << len(dut.rd_base_row_o)
K = TOPK_K
query = (1 << HASH_BITS) - 1
dut.query_hash.value = query
dut.query_valid.value = 1
await RisingEdge(dut.clk)
dut.query_valid.value = 0
await feed_all_batches(dut)
# Wait for result completion
await collect_serial_results(dut, K, timeout_cycles=800)
# After completion, consumed_candidates_debug should show all rows consumed
consumed = int(dut.consumed_candidates_debug.value)
assert consumed == NUM_ROWS, (
f"consumed_candidates_debug={consumed} != NUM_ROWS={NUM_ROWS}"
)
# ==============================================================================
# Test: K=1 regression
# ==============================================================================
@cocotb.test()
async def match_engine_topk_k1(dut):
"""K=1: verify single beat with rank=0, last=1, row/score matches golden.
Feeds all rows all-lane-valid with a single high-score row (row 9).
Only one result beat is expected; it must carry rank=0, last=1, and
match the golden Top-1 (row, score).
"""
if TOPK_K != 1:
dut._log.info(f"Skipping: K=1 test requires TOPK_K=1 (got {TOPK_K})")
return
cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start())
await reset_match(dut)
LANES = len(dut.rd_lane_valid_i)
ROW_BITS = len(dut.rd_row_ids_i) // LANES
HASH_BITS = len(dut.rd_hashes_i) // LANES
NUM_ROWS = 1 << len(dut.rd_base_row_o)
query = (1 << HASH_BITS) - 1
# Row 9 gets the best hash; all others get 0
special: dict[int, int] = {9: query}
rows_and_hashes: list[tuple[int, int]] = []
dut.query_hash.value = query
dut.query_valid.value = 1
await RisingEdge(dut.clk)
dut.query_valid.value = 0
for base in range(0, NUM_ROWS, LANES):
while int(dut.rd_valid_o.value) == 0:
await RisingEdge(dut.clk)
rows = 0
hashes = 0
for lane in range(LANES):
row_id = base + lane
row_hash = special.get(row_id, 0)
rows_and_hashes.append((row_id, row_hash))
rows |= row_id << (lane * ROW_BITS)
hashes |= row_hash << (lane * HASH_BITS)
dut.rd_row_ids_i.value = rows
dut.rd_hashes_i.value = hashes
dut.rd_lane_valid_i.value = (1 << LANES) - 1
dut.rd_valid_i.value = 1
await RisingEdge(dut.clk)
dut.rd_valid_i.value = 0
expected = golden_topk(rows_and_hashes, query, HASH_BITS, 1)
assert len(expected) == 1
exp_row, exp_score = expected[0]
# Collect single result
dut.result_ready.value = 1
for _ in range(200):
if int(dut.result_valid.value):
rank = int(dut.result_rank.value)
row = int(dut.result_row.value)
score = int(dut.result_score.value)
last = int(dut.result_last.value)
assert rank == 0, f"rank={rank} != 0"
assert last == 1, f"last={last} != 1"
assert row == exp_row, f"row={row} != {exp_row}"
assert score == exp_score, f"score={score} != {exp_score}"
return
await RisingEdge(dut.clk)
raise AssertionError("result_valid never went high")