From e4cbb5e30d623f4aa59471d0ced6096a6cbb073e Mon Sep 17 00:00:00 2001 From: SikongJueluo Date: Tue, 19 May 2026 18:19:05 +0800 Subject: [PATCH] feat(hw/rtl): implement full Top-K CAM search pipeline with serial result output - add TOPK_K, FIFO_DEPTH, RESULT_SERIAL parameters to cam_params - add candidate_fifo: synchronous ready/valid FIFO for (row, score) candidates - add topk_tracker: tracks top-K candidates with clear/ready handshake - add result_serializer: serializes packed Top-K array into rank-ordered stream - refactor match_engine_pipeline from 4-state to 8-state Top-K pipeline - extend cam_top with serial Top-K interface (result_rank/row/score/last) - add backward-compatible top1_index/top1_score aliases from rank-0 beat - add comprehensive tests for all new modules --- hw/rtl/cam_params.svh | 18 + hw/rtl/cam_top.sv | 29 +- hw/rtl/core/candidate_fifo.sv | 124 ++++ hw/rtl/core/match_engine_pipeline.sv | 451 +++++++++++++-- hw/rtl/core/result_serializer.sv | 126 +++++ hw/rtl/core/topk_tracker.sv | 126 +++++ hw/sim/Makefile | 2 +- hw/sim/mk/rtl-sources.mk | 3 + hw/sim/tests/modules/candidate_fifo/Makefile | 32 ++ .../tests/modules/candidate_fifo/__init__.py | 0 .../candidate_fifo/test_candidate_fifo.py | 306 ++++++++++ .../modules/match_engine_pipeline/Makefile | 20 + .../test_match_engine_pipeline.py | 531 +++++++++++++++++- .../tests/modules/result_serializer/Makefile | 29 + .../test_result_serializer.py | 212 +++++++ hw/sim/tests/modules/topk_tracker/Makefile | 9 + .../modules/topk_tracker/test_topk_tracker.py | 133 +++++ hw/sim/tests/top/test_cam_basic.py | 115 ++-- hw/syn/Makefile | 4 +- hw/syn/synth_cam_top_flat.ys | 3 + hw/syn/synth_cam_top_hier.ys | 3 + 21 files changed, 2152 insertions(+), 124 deletions(-) create mode 100644 hw/rtl/core/candidate_fifo.sv create mode 100644 hw/rtl/core/result_serializer.sv create mode 100644 hw/rtl/core/topk_tracker.sv create mode 100644 hw/sim/tests/modules/candidate_fifo/Makefile create mode 100644 hw/sim/tests/modules/candidate_fifo/__init__.py create mode 100644 hw/sim/tests/modules/candidate_fifo/test_candidate_fifo.py create mode 100644 hw/sim/tests/modules/result_serializer/Makefile create mode 100644 hw/sim/tests/modules/result_serializer/test_result_serializer.py create mode 100644 hw/sim/tests/modules/topk_tracker/Makefile create mode 100644 hw/sim/tests/modules/topk_tracker/test_topk_tracker.py diff --git a/hw/rtl/cam_params.svh b/hw/rtl/cam_params.svh index cf62025..ce42d22 100644 --- a/hw/rtl/cam_params.svh +++ b/hw/rtl/cam_params.svh @@ -9,6 +9,9 @@ // NUM_ROWS — Number of rows in the CAM array // HASH_BITS — Width of the hash value in bits // LANES — Number of parallel comparison lanes +// TOPK_K — Number of best matches retained per query +// FIFO_DEPTH — Candidate FIFO entries between match engine and tracker +// RESULT_SERIAL — Enables rank-ordered serial result output // ROW_BITS — Bits needed to index NUM_ROWS rows ($clog2(NUM_ROWS)) // SCORE_BITS — Bits needed to represent HASH_BITS-bit scores ($clog2(HASH_BITS+1)) // @@ -34,6 +37,21 @@ `define LANES 8 `endif +// Number of Top-K results to retain and serialize. +`ifndef TOPK_K +`define TOPK_K 4 +`endif + +// Candidate FIFO depth between score generation and Top-K tracking. +`ifndef FIFO_DEPTH +`define FIFO_DEPTH 16 +`endif + +// Result interface mode. First implementation uses serial Top-K output. +`ifndef RESULT_SERIAL +`define RESULT_SERIAL 1 +`endif + // Bits required to represent a row index. `ifndef ROW_BITS `define ROW_BITS $clog2(`NUM_ROWS) diff --git a/hw/rtl/cam_top.sv b/hw/rtl/cam_top.sv index 3002b5d..621e02c 100644 --- a/hw/rtl/cam_top.sv +++ b/hw/rtl/cam_top.sv @@ -27,14 +27,20 @@ module cam_top #( output logic query_ready, input logic [(`HASH_BITS)-1:0] query_hash, - // Result interface + // Result interface (serial Top-K) output logic result_valid, input logic result_ready, + output logic [((`TOPK_K <= 1) ? 1 : $clog2(`TOPK_K))-1:0] result_rank, + output logic [(`ROW_BITS)-1:0] result_row, + output logic [(`SCORE_BITS)-1:0] result_score, + output logic result_last, + // Legacy Top-1 aliases (captured from rank-0 serial beat) output logic [(`ROW_BITS)-1:0] top1_index, output logic [(`SCORE_BITS)-1:0] top1_score, `ifdef SIM_DEBUG output logic [(`NUM_ROWS)*(`SCORE_BITS)-1:0] score_debug_flat, + output logic [$clog2(`NUM_ROWS+1)-1:0] consumed_candidates_debug, `endif output logic busy @@ -101,8 +107,10 @@ module cam_top #( .query_hash (query_hash), .result_valid (result_valid), .result_ready (result_ready), - .result_row (top1_index), - .result_score (top1_score), + .result_rank (result_rank), + .result_row (result_row), + .result_score (result_score), + .result_last (result_last), .busy (match_busy), .rd_valid_o (rd_req_valid), .rd_base_row_o (rd_req_base_row), @@ -111,8 +119,21 @@ module cam_top #( .rd_hashes_i (rd_resp_hashes), .rd_lane_valid_i (rd_resp_lane_valid) `ifdef SIM_DEBUG - ,.score_debug_flat (score_debug_flat) + ,.score_debug_flat (score_debug_flat) + ,.consumed_candidates_debug (consumed_candidates_debug) `endif ); + // ── Rank-0 (Top-1) aliases ── + // Capture first serial beat (rank==0) into legacy top1_index/top1_score outputs. + always_ff @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + top1_index <= '0; + top1_score <= '0; + end else if (result_valid && result_ready && (result_rank == '0)) begin + top1_index <= result_row; + top1_score <= result_score; + end + end + endmodule diff --git a/hw/rtl/core/candidate_fifo.sv b/hw/rtl/core/candidate_fifo.sv new file mode 100644 index 0000000..ae585e2 --- /dev/null +++ b/hw/rtl/core/candidate_fifo.sv @@ -0,0 +1,124 @@ +`timescale 1ns / 1ps +`include "cam_params.svh" + +//============================================================================== +// candidate_fifo — Synchronous ready/valid FIFO for (row, score) candidates +// +// Implements a simple circular-buffer FIFO that accepts writes only when +// wr_valid_i && wr_ready_o and reads only when rd_valid_o && rd_ready_i. +// Provides empty/full flags. Default depth is `FIFO_DEPTH (16). +// +// Quality: +// - Pointers always wrap at DEPTH-1 (safe for non-power-of-two depths). +// - wr_ready_o goes high when full if a read is accepted in the same cycle, +// allowing a simultaneous write (throughput-friendly backpressure). +// - PTR_BITS is at least 1 for DEPTH=1. +//============================================================================== + +module candidate_fifo #( + parameter int DEPTH = `FIFO_DEPTH, + parameter int ROW_BITS = `ROW_BITS, + parameter int SCORE_BITS = `SCORE_BITS, + // Ensure at least 1 bit even when DEPTH=1. + parameter int PTR_BITS = (DEPTH <= 1) ? 1 : $clog2(DEPTH) +) ( + input logic clk, + input logic rst_n, + + // Write interface (producer side) + input logic wr_valid_i, + output logic wr_ready_o, + input logic [ROW_BITS -1:0] wr_row_i, + input logic [SCORE_BITS -1:0] wr_score_i, + + // Read interface (consumer side) + output logic rd_valid_o, + input logic rd_ready_i, + output logic [ROW_BITS -1:0] rd_row_o, + output logic [SCORE_BITS -1:0] rd_score_o, + + // Status flags + output logic empty_o, + output logic full_o +); + + //-------------------------------------------------------------------------- + // Storage — dual‑port array. + //-------------------------------------------------------------------------- + logic [ROW_BITS -1:0] mem_row [0:DEPTH-1]; + logic [SCORE_BITS -1:0] mem_score [0:DEPTH-1]; + + //-------------------------------------------------------------------------- + // Pointers and fill counter + //-------------------------------------------------------------------------- + logic [PTR_BITS -1:0] wr_ptr, rd_ptr; + logic [PTR_BITS :0] count; // extra bit for full detect + + // Wrap-at value (DEPTH-1), sized to match pointer width. + // Part-select DEPTH and use a 1'b1 literal so the RHS width equals PTR_BITS. + localparam logic [PTR_BITS-1:0] WRAP_AT = DEPTH[PTR_BITS-1:0] - 1'b1; + + //-------------------------------------------------------------------------- + // Status flag outputs (combinational) + //-------------------------------------------------------------------------- + assign empty_o = (count == 0); + assign full_o = (count == DEPTH[PTR_BITS:0]); + + //-------------------------------------------------------------------------- + // Handshake outputs + // + // wr_ready_o is asserted when not full, OR when a read is accepted in + // the same cycle (full+pop → slot freed → write can proceed). + //-------------------------------------------------------------------------- + assign wr_ready_o = ~full_o | (rd_valid_o & rd_ready_i); + assign rd_valid_o = ~empty_o; + + //-------------------------------------------------------------------------- + // Read data (combinational — value at current rd_ptr) + //-------------------------------------------------------------------------- + assign rd_row_o = mem_row [rd_ptr]; + assign rd_score_o = mem_score[rd_ptr]; + + //-------------------------------------------------------------------------- + // Push / Pop decode + //-------------------------------------------------------------------------- + logic push, pop; + assign push = wr_valid_i & wr_ready_o; + assign pop = rd_valid_o & rd_ready_i; + + //-------------------------------------------------------------------------- + // Sequential — pointer and count updates + // + // Pointers wrap at DEPTH-1, safe for any DEPTH (not just powers of two). + // Simultaneous push+pop leaves count unchanged. + //-------------------------------------------------------------------------- + always_ff @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + wr_ptr <= 0; + rd_ptr <= 0; + count <= 0; + end else begin + if (push) begin + mem_row [wr_ptr] <= wr_row_i; + mem_score[wr_ptr] <= wr_score_i; + if (wr_ptr == WRAP_AT) + wr_ptr <= 0; + else + wr_ptr <= wr_ptr + 1; + end + + if (pop) begin + if (rd_ptr == WRAP_AT) + rd_ptr <= 0; + else + rd_ptr <= rd_ptr + 1; + end + + // Update fill count + if (push & ~pop) count <= count + 1; + else if (~push & pop) count <= count - 1; + // else: no change (neither, or both) + end + end + +endmodule diff --git a/hw/rtl/core/match_engine_pipeline.sv b/hw/rtl/core/match_engine_pipeline.sv index 481a89e..83bd0d1 100644 --- a/hw/rtl/core/match_engine_pipeline.sv +++ b/hw/rtl/core/match_engine_pipeline.sv @@ -7,10 +7,15 @@ module match_engine_pipeline ( input logic query_valid, output logic query_ready, input logic [(`HASH_BITS)-1:0] query_hash, + + // Serial Top-K result interface output logic result_valid, input logic result_ready, + output logic [((`TOPK_K <= 1) ? 1 : $clog2(`TOPK_K))-1:0] result_rank, output logic [(`ROW_BITS)-1:0] result_row, output logic [(`SCORE_BITS)-1:0] result_score, + output logic result_last, + output logic busy, output logic rd_valid_o, output logic [(`ROW_BITS)-1:0] rd_base_row_o, @@ -20,35 +25,150 @@ module match_engine_pipeline ( input logic [(`LANES)-1:0] rd_lane_valid_i `ifdef SIM_DEBUG ,output logic [(`NUM_ROWS)*(`SCORE_BITS)-1:0] score_debug_flat + ,output logic [($clog2(`NUM_ROWS+1))-1:0] consumed_candidates_debug `endif ); - typedef enum logic [1:0] {S_IDLE, S_SCAN, S_DRAIN, S_DONE} state_t; + //========================================================================== + // State encoding + //========================================================================== + typedef enum logic [2:0] { + S_IDLE, + S_ISSUE_READ, + S_WAIT_SCORE, + S_STAGE_CANDIDATES, + S_PUSH_CANDIDATES, + S_DRAIN, + S_SERIALIZE_RESULT, + S_DONE + } state_t; + state_t state_q; + //========================================================================== + // Query / read tracking + //========================================================================== logic [(`HASH_BITS)-1:0] query_q; - logic [(`ROW_BITS)-1:0] issue_base_q; - logic [$clog2(`NUM_ROWS/`LANES+1)-1:0] returned_q; - logic [(`ROW_BITS)-1:0] best_row_q; - logic [(`SCORE_BITS)-1:0] best_score_q; + logic [(`ROW_BITS)-1:0] issue_base_q; + logic [$clog2(`NUM_ROWS/`LANES+1)-1:0] issued_batches_q; + logic [$clog2(`NUM_ROWS/`LANES+1)-1:0] returned_batches_q; + logic [$clog2(`NUM_ROWS+1)-1:0] pushed_candidates_q; + logic [$clog2(`NUM_ROWS+1)-1:0] consumed_candidates_q; + // Total expected valid candidates across all batches + logic [$clog2(`NUM_ROWS+1)-1:0] expected_candidates_q; + // Batch-level valid-lane mask for return tracking + logic [(`LANES)-1:0] batch_lane_mask_q; + //-------------------------------------------------------------------------- + // Read/noise staging registers (feed popcount pipeline) + //-------------------------------------------------------------------------- + logic rd_stage_valid_q; + logic [(`LANES)*(`ROW_BITS)-1:0] rd_stage_row_ids_q; + logic [(`LANES)*(`HASH_BITS)-1:0] rd_stage_hashes_q; + logic [(`LANES)-1:0] rd_stage_lane_valid_q; + + //-------------------------------------------------------------------------- + // Candidate staging registers (scored output from popcount, before FIFO) + //-------------------------------------------------------------------------- + logic [(`LANES)-1:0] staged_valid_q; + logic [(`LANES)*(`ROW_BITS)-1:0] staged_rows_q; + logic [(`LANES)*(`SCORE_BITS)-1:0] staged_scores_q; + logic [$clog2(`LANES)-1:0] push_lane_idx_q; + + //========================================================================== + // Popcount per-lane wires + //========================================================================== logic [(`HASH_BITS)-1:0] match_bits [0:`LANES-1]; logic score_valid [0:`LANES-1]; logic [(`ROW_BITS)-1:0] score_row [0:`LANES-1]; logic [(`SCORE_BITS)-1:0] lane_score [0:`LANES-1]; - logic [(`ROW_BITS)-1:0] batch_row; - logic [(`SCORE_BITS)-1:0] batch_score; - assign query_ready = (state_q == S_IDLE); - assign result_valid = (state_q == S_DONE); - assign result_row = best_row_q; - assign result_score = best_score_q; - assign busy = (state_q != S_IDLE); - assign rd_valid_o = (state_q == S_SCAN); - assign rd_base_row_o = issue_base_q; + //========================================================================== + // Candidate FIFO interface + //========================================================================== + logic cand_valid; + logic cand_ready; + logic [(`ROW_BITS)-1:0] cand_row; + logic [(`SCORE_BITS)-1:0] cand_score; + wire cand_fire = cand_valid && cand_ready; + //========================================================================== + // FIFO wires + //========================================================================== + logic fifo_wr_valid; + logic fifo_wr_ready; + logic [(`ROW_BITS)-1:0] fifo_wr_row; + logic [(`SCORE_BITS)-1:0] fifo_wr_score; + logic fifo_rd_valid; + logic fifo_rd_ready; + logic [(`ROW_BITS)-1:0] fifo_rd_row; + logic [(`SCORE_BITS)-1:0] fifo_rd_score; + logic fifo_empty; + + //========================================================================== + // Tracker wires + //========================================================================== + logic tracker_clear; + logic tracker_candidate_ready; + logic [(`TOPK_K)*(`ROW_BITS)-1:0] tracker_topk_rows; + logic [(`TOPK_K)*(`SCORE_BITS)-1:0] tracker_topk_scores; + logic tracker_update_pending; + + //========================================================================== + // Serializer wires + //========================================================================== + logic serializer_start; + logic serializer_busy; + logic serializer_done; + logic serializer_result_valid; + logic [((`TOPK_K <= 1) ? 1 : $clog2(`TOPK_K))-1:0] serializer_result_rank; + logic [(`ROW_BITS)-1:0] serializer_result_row; + logic [(`SCORE_BITS)-1:0] serializer_result_score; + logic serializer_result_last; + + //========================================================================== + // Completion condition wires + // Width-matched comparison targets to suppress Verilator WIDTHEXPAND. + // The assigned constants fit exactly — WIDTHTRUNC is benign. + /* verilator lint_off WIDTHTRUNC */ + wire [$clog2(`NUM_ROWS/`LANES+1)-1:0] num_batches_w = `NUM_ROWS / `LANES; + wire [$clog2(`LANES)-1:0] last_lane_w = `LANES - 1; + /* verilator lint_on WIDTHTRUNC */ + + wire all_reads_issued = (issued_batches_q == num_batches_w); + wire all_scores_returned = (returned_batches_q == num_batches_w); + wire all_candidates_pushed = (pushed_candidates_q == expected_candidates_q); + wire all_candidates_consumed = (consumed_candidates_q == expected_candidates_q); + wire scan_drained = all_reads_issued && all_scores_returned + && all_candidates_pushed + && all_candidates_consumed + && fifo_empty + && !tracker_update_pending; + + //========================================================================== + // Combinational output assignments + //========================================================================== + assign query_ready = (state_q == S_IDLE); + assign rd_valid_o = (state_q == S_ISSUE_READ); + assign rd_base_row_o = issue_base_q; + assign busy = (state_q != S_IDLE) || serializer_busy; + +`ifdef SIM_DEBUG + assign consumed_candidates_debug = consumed_candidates_q; +`endif + + // Result outputs come directly from the serializer + assign result_valid = serializer_result_valid; + assign result_rank = serializer_result_rank; + assign result_row = serializer_result_row; + assign result_score = serializer_result_score; + assign result_last = serializer_result_last; + + //========================================================================== + // Generate: popcount pipeline per lane + //========================================================================== generate for (genvar lane = 0; lane < `LANES; lane++) begin : gen_scores - assign match_bits[lane] = ~(query_q ^ rd_hashes_i[lane*`HASH_BITS +: `HASH_BITS]); + assign match_bits[lane] = ~(query_q ^ rd_stage_hashes_q[lane*`HASH_BITS +: `HASH_BITS]); popcount_pipeline #( .WIDTH(`HASH_BITS), .ROW_BITS(`ROW_BITS), @@ -56,8 +176,8 @@ module match_engine_pipeline ( ) u_popcount_pipeline ( .clk(clk), .rst_n(rst_n), - .valid_i(rd_valid_i && rd_lane_valid_i[lane]), - .row_i(rd_row_ids_i[lane*`ROW_BITS +: `ROW_BITS]), + .valid_i(rd_stage_valid_q && rd_stage_lane_valid_q[lane]), + .row_i(rd_stage_row_ids_q[lane*`ROW_BITS +: `ROW_BITS]), .bits_i(match_bits[lane]), .valid_o(score_valid[lane]), .row_o(score_row[lane]), @@ -66,75 +186,284 @@ module match_engine_pipeline ( end endgenerate - always_comb begin - batch_row = score_row[0]; - batch_score = lane_score[0]; - for (int lane = 1; lane < `LANES; lane++) begin - if ((lane_score[lane] > batch_score) || - ((lane_score[lane] == batch_score) && (score_row[lane] < batch_row))) begin - batch_score = lane_score[lane]; - batch_row = score_row[lane]; + //========================================================================== + // Batch scores done: true when all valid lanes in the current batch + // have their score_valid asserted, or vacuously true for an all-invalid batch. + wire batch_scores_done; + generate + if (`LANES == 1) begin : gen_batch_done_1 + assign batch_scores_done = rd_stage_lane_valid_q[0] + ? score_valid[0] : 1'b1; + end else begin : gen_batch_done_n + logic all_valid_ready; + always_comb begin + all_valid_ready = 1'b1; + for (int l = 0; l < `LANES; l++) begin + if (rd_stage_lane_valid_q[l] && !score_valid[l]) begin + all_valid_ready = 1'b0; + end + end end + assign batch_scores_done = all_valid_ready; end - end + endgenerate + + //========================================================================== + // Candidate FIFO write interface + //========================================================================== + assign cand_valid = (state_q == S_PUSH_CANDIDATES) + && staged_valid_q[push_lane_idx_q]; + assign cand_row = staged_rows_q[push_lane_idx_q * `ROW_BITS +: `ROW_BITS]; + assign cand_score = staged_scores_q[push_lane_idx_q * `SCORE_BITS +: `SCORE_BITS]; + assign cand_ready = fifo_wr_ready; + + assign fifo_wr_valid = cand_valid; + assign fifo_wr_row = cand_row; + assign fifo_wr_score = cand_score; + + //========================================================================== + // FIFO output + //========================================================================== + assign fifo_rd_ready = tracker_candidate_ready; + + wire fifo_rd_fire = fifo_rd_valid && fifo_rd_ready; + + //========================================================================== + // Serializer start edge detection + //========================================================================== + logic start_ser_q1, start_ser_q2; + wire start_ser_raw = (state_q == S_DRAIN) && scan_drained; always_ff @(posedge clk or negedge rst_n) begin if (!rst_n) begin - state_q <= S_IDLE; - query_q <= '0; - issue_base_q <= '0; - returned_q <= 0; - best_row_q <= '0; - best_score_q <= '0; + start_ser_q1 <= 1'b0; + start_ser_q2 <= 1'b0; + end else begin + start_ser_q1 <= start_ser_raw; + start_ser_q2 <= start_ser_q1; + end + end + + assign serializer_start = start_ser_q1 && !start_ser_q2; + + //========================================================================== + // Tracker clear: pulse on new query + //========================================================================== + assign tracker_clear = (state_q == S_IDLE) && query_valid; + + //========================================================================== + // Submodule instantiations + //========================================================================== + + // --- candidate_fifo (full_o unused) --- + /* verilator lint_off PINCONNECTEMPTY */ + candidate_fifo #( + .DEPTH(`FIFO_DEPTH), + .ROW_BITS(`ROW_BITS), + .SCORE_BITS(`SCORE_BITS) + ) u_candidate_fifo ( + .clk(clk), + .rst_n(rst_n), + .wr_valid_i(fifo_wr_valid), + .wr_ready_o(fifo_wr_ready), + .wr_row_i(fifo_wr_row), + .wr_score_i(fifo_wr_score), + .rd_valid_o(fifo_rd_valid), + .rd_ready_i(fifo_rd_ready), + .rd_row_o(fifo_rd_row), + .rd_score_o(fifo_rd_score), + .empty_o(fifo_empty), + .full_o() // intentionally unused + ); + /* verilator lint_on PINCONNECTEMPTY */ + + // --- topk_tracker --- + topk_tracker #( + .K(`TOPK_K), + .ROW_BITS(`ROW_BITS), + .SCORE_BITS(`SCORE_BITS) + ) u_topk_tracker ( + .clk(clk), + .rst_n(rst_n), + .clear_i(tracker_clear), + .candidate_valid_i(fifo_rd_valid), + .candidate_ready_o(tracker_candidate_ready), + .candidate_row_i(fifo_rd_row), + .candidate_score_i(fifo_rd_score), + .topk_rows_o(tracker_topk_rows), + .topk_scores_o(tracker_topk_scores), + .update_pending_o(tracker_update_pending) + ); + + // --- result_serializer --- + result_serializer #( + .K(`TOPK_K), + .ROW_BITS(`ROW_BITS), + .SCORE_BITS(`SCORE_BITS) + ) u_result_serializer ( + .clk(clk), + .rst_n(rst_n), + .start_i(serializer_start), + .busy_o(serializer_busy), + .done_o(serializer_done), + .topk_rows_i(tracker_topk_rows), + .topk_scores_i(tracker_topk_scores), + .result_valid_o(serializer_result_valid), + .result_ready_i(result_ready), + .result_rank_o(serializer_result_rank), + .result_row_o(serializer_result_row), + .result_score_o(serializer_result_score), + .result_last_o(serializer_result_last) + ); + + //========================================================================== + // Sequential: main FSM + registers + //========================================================================== + always_ff @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + state_q <= S_IDLE; + query_q <= '0; + issue_base_q <= '0; + issued_batches_q <= 0; + returned_batches_q <= 0; + pushed_candidates_q <= 0; + consumed_candidates_q <= 0; + expected_candidates_q <= 0; + batch_lane_mask_q <= '0; + rd_stage_valid_q <= 1'b0; + rd_stage_row_ids_q <= '0; + rd_stage_hashes_q <= '0; + rd_stage_lane_valid_q <= '0; + staged_valid_q <= '0; + staged_rows_q <= '0; + staged_scores_q <= '0; + push_lane_idx_q <= 0; `ifdef SIM_DEBUG - score_debug_flat <= '0; + score_debug_flat <= '0; `endif end else begin + // Defaults for one-cycle flags + rd_stage_valid_q <= 1'b0; + + // Consumed candidates counter (FIFO → Tracker handshake) + if (fifo_rd_fire) begin + consumed_candidates_q <= consumed_candidates_q + 1; + end + + // State machine unique case (state_q) + S_IDLE: begin if (query_valid) begin - query_q <= query_hash; - issue_base_q <= '0; - returned_q <= 0; - best_row_q <= TIE_BREAK_SENTINEL; - best_score_q <= '0; + query_q <= query_hash; + issue_base_q <= '0; + issued_batches_q <= 0; + returned_batches_q <= 0; + pushed_candidates_q <= 0; + consumed_candidates_q <= 0; + expected_candidates_q <= 0; + batch_lane_mask_q <= '0; + push_lane_idx_q <= 0; + staged_valid_q <= '0; `ifdef SIM_DEBUG - score_debug_flat <= '0; + score_debug_flat <= '0; `endif - state_q <= S_SCAN; + state_q <= S_ISSUE_READ; end end - S_SCAN: begin - if (issue_base_q + `LANES >= `NUM_ROWS) begin - state_q <= S_DRAIN; - end else begin - issue_base_q <= issue_base_q + `LANES; + + S_ISSUE_READ: begin + if (rd_valid_i) begin + // Capture staging data for popcount pipeline + rd_stage_valid_q <= 1'b1; + rd_stage_row_ids_q <= rd_row_ids_i; + rd_stage_hashes_q <= rd_hashes_i; + rd_stage_lane_valid_q <= rd_lane_valid_i; + // Remember the lane mask for this batch (used in S_WAIT_SCORE + // to determine batch_scores_done — survives until overwritten + // by the next S_ISSUE_READ). + batch_lane_mask_q <= rd_lane_valid_i; + issue_base_q <= issue_base_q + `LANES; + issued_batches_q <= issued_batches_q + 1; + // Accumulate total expected valid candidates + expected_candidates_q <= expected_candidates_q + + $countones(rd_lane_valid_i); + state_q <= S_WAIT_SCORE; end end + + S_WAIT_SCORE: begin + // rd_stage_valid_q already deasserted by default above + if (batch_scores_done) begin + returned_batches_q <= returned_batches_q + 1; + + // Capture per-lane scores and rows into staging registers + for (int lane = 0; lane < `LANES; lane++) begin + if (batch_lane_mask_q[lane]) begin + staged_valid_q[lane] <= 1'b1; + staged_rows_q[lane*`ROW_BITS +: `ROW_BITS] <= score_row[lane]; + staged_scores_q[lane*`SCORE_BITS +: `SCORE_BITS] <= lane_score[lane]; + end else begin + staged_valid_q[lane] <= 1'b0; + end + end + +`ifdef SIM_DEBUG + for (int lane = 0; lane < `LANES; lane++) begin + if (batch_lane_mask_q[lane]) begin + score_debug_flat[score_row[lane]*`SCORE_BITS +: `SCORE_BITS] + <= lane_score[lane]; + end + end +`endif + state_q <= S_STAGE_CANDIDATES; + end + end + + S_STAGE_CANDIDATES: begin + state_q <= S_PUSH_CANDIDATES; + end + + S_PUSH_CANDIDATES: begin + // Advance if the current lane fired or wasn't valid + if (cand_fire || !staged_valid_q[push_lane_idx_q]) begin + if (cand_fire) begin + pushed_candidates_q <= pushed_candidates_q + 1; + end + + if (push_lane_idx_q == last_lane_w) begin + // Last lane processed — decide next state + push_lane_idx_q <= 0; + if (all_reads_issued) begin + state_q <= S_DRAIN; + end else begin + state_q <= S_ISSUE_READ; + end + end else begin + push_lane_idx_q <= push_lane_idx_q + 1; + end + end + end + S_DRAIN: begin - if ((returned_q == (`NUM_ROWS / `LANES)) && !score_valid[0]) begin + if (scan_drained) begin + state_q <= S_SERIALIZE_RESULT; + end + end + + S_SERIALIZE_RESULT: begin + if (serializer_done) begin state_q <= S_DONE; end end + S_DONE: begin - if (result_ready) state_q <= S_IDLE; + state_q <= S_IDLE; end + default: state_q <= S_IDLE; endcase - - if (score_valid[0]) begin - returned_q <= returned_q + 1'b1; - if ((batch_score > best_score_q) || - ((batch_score == best_score_q) && (batch_row < best_row_q))) begin - best_score_q <= batch_score; - best_row_q <= batch_row; - end -`ifdef SIM_DEBUG - for (int lane = 0; lane < `LANES; lane++) begin - score_debug_flat[score_row[lane]*`SCORE_BITS +: `SCORE_BITS] <= lane_score[lane]; - end -`endif - end end end + endmodule diff --git a/hw/rtl/core/result_serializer.sv b/hw/rtl/core/result_serializer.sv new file mode 100644 index 0000000..b681182 --- /dev/null +++ b/hw/rtl/core/result_serializer.sv @@ -0,0 +1,126 @@ +`timescale 1ns / 1ps +`include "cam_params.svh" + +//============================================================================== +// Result Serializer +//============================================================================== +// Serializes the packed Top-K array (rows and scores) into a rank-ordered +// handshake interface, one result per fire. +// +// Start is edge-triggered (sampled on the cycle start_i is high while not +// already active). Once started, the module presents outputs for the +// current rank (starting at 0) and advances on each valid-ready fire. +// After the last rank (K-1) fires, done_o pulses for one cycle and the +// module returns to idle with rank reset to 0. +// +// Back-pressure: holds current output while result_ready_i is low. +// +// Supports K=1 cleanly (last rank = rank 0, done on first fire). +//============================================================================== + +module result_serializer #( + parameter int K = `TOPK_K, + parameter int ROW_BITS = `ROW_BITS, + parameter int SCORE_BITS = `SCORE_BITS, + parameter int RANK_BITS = (K <= 1) ? 1 : $clog2(K) +) ( + input logic clk, + input logic rst_n, + + input logic start_i, + output logic busy_o, + output logic done_o, + + input logic [K*ROW_BITS-1:0] topk_rows_i, + input logic [K*SCORE_BITS-1:0] topk_scores_i, + + output logic result_valid_o, + input logic result_ready_i, + output logic [RANK_BITS-1:0] result_rank_o, + output logic [ROW_BITS-1:0] result_row_o, + output logic [SCORE_BITS-1:0] result_score_o, + output logic result_last_o +); + + // ── State encoding ───────────────────────────────────────────────────── + typedef enum logic { + IDLE, + ACTIVE + } state_t; + + // ── Localparam for last-rank comparison (width-matched to avoid warnings) ─ + localparam [RANK_BITS-1:0] LAST_RANK = RANK_BITS'(K - 1); + + state_t state_q, state_next; + + logic [RANK_BITS-1:0] rank_q, rank_next; + logic done_q, done_next; + + // Snapshot of packed inputs captured at start (holds during back-pressure) + logic [K*ROW_BITS-1:0] rows_snapshot_q, rows_snapshot_next; + logic [K*SCORE_BITS-1:0] scores_snapshot_q, scores_snapshot_next; + + // ── Sequential: state, rank, done, and snapshot registers ───────────── + always_ff @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + state_q <= IDLE; + rank_q <= '0; + done_q <= 1'b0; + rows_snapshot_q <= '0; + scores_snapshot_q <= '0; + end else begin + state_q <= state_next; + rank_q <= rank_next; + done_q <= done_next; + rows_snapshot_q <= rows_snapshot_next; + scores_snapshot_q <= scores_snapshot_next; + end + end + + // ── Combinational: next-state, snapshot, and output logic ────────────── + always_comb begin + // Defaults: stay in current state, retain rank, no done pulse, + // hold snapshot (outputs stable under back-pressure) + state_next = state_q; + rank_next = rank_q; + done_next = 1'b0; + rows_snapshot_next = rows_snapshot_q; + scores_snapshot_next = scores_snapshot_q; + + if (state_q == IDLE) begin + // Edge-triggered start — capture packed inputs into snapshot + if (start_i) begin + state_next = ACTIVE; + rank_next = '0; // start from rank 0 + rows_snapshot_next = topk_rows_i; + scores_snapshot_next = topk_scores_i; + end + end else begin + // ACTIVE: advance on fire + if (result_valid_o && result_ready_i) begin + // Is this the last rank? + if (rank_q == LAST_RANK) begin + state_next = IDLE; + rank_next = '0; + done_next = 1'b1; + end else begin + rank_next = rank_q + 1'b1; + end + end + end + end + + // ── Output assignments ───────────────────────────────────────────────── + assign busy_o = (state_q == ACTIVE); + assign done_o = done_q; // registered — pulses after deactivation + assign result_valid_o = (state_q == ACTIVE); + + // Slice the snapshot registers at the current rank + assign result_row_o = rows_snapshot_q[rank_q * ROW_BITS +: ROW_BITS]; + assign result_score_o = scores_snapshot_q[rank_q * SCORE_BITS +: SCORE_BITS]; + assign result_rank_o = rank_q; + + // result_last_o: high only when active and on the last rank + assign result_last_o = (state_q == ACTIVE) && (rank_q == LAST_RANK); + +endmodule diff --git a/hw/rtl/core/topk_tracker.sv b/hw/rtl/core/topk_tracker.sv new file mode 100644 index 0000000..ef57f9f --- /dev/null +++ b/hw/rtl/core/topk_tracker.sv @@ -0,0 +1,126 @@ +`timescale 1ns / 1ps +`include "cam_params.svh" + +//============================================================================== +// Top-K Tracker +//============================================================================== +// Maintains a sorted Top-K array of (row, score) candidates. +// Insertion sort on every candidate handshake: +// - Higher score wins. +// - If equal score, lower row wins (tie-break). +// - Entries that are worse than all K current slots are dropped. +// +// candidate_ready_o is always 1 — the tracker accepts a candidate every cycle. +// update_pending_o is always 0 — insertion is fully combinational (one cycle). +// +// Packed outputs: +// topk_rows_o [i*ROW_BITS +: ROW_BITS] = rank i row +// topk_scores_o [i*SCORE_BITS +: SCORE_BITS] = rank i score +//============================================================================== + +module topk_tracker #( + parameter int K = `TOPK_K, + parameter int ROW_BITS = `ROW_BITS, + parameter int SCORE_BITS = `SCORE_BITS +) ( + input logic clk, + input logic rst_n, + input logic clear_i, + + input logic candidate_valid_i, + output logic candidate_ready_o, + input logic [ROW_BITS-1:0] candidate_row_i, + input logic [SCORE_BITS-1:0] candidate_score_i, + + output logic [K*ROW_BITS-1:0] topk_rows_o, + output logic [K*SCORE_BITS-1:0] topk_scores_o, + output logic update_pending_o +); + + // ── Sequential state: K-entry sorted arrays ──────────────────────────── + logic [ROW_BITS-1:0] rows_q [0:K-1]; + logic [SCORE_BITS-1:0] scores_q [0:K-1]; + + // ── Next-state signals ───────────────────────────────────────────────── + logic [ROW_BITS-1:0] rows_next [0:K-1]; + logic [SCORE_BITS-1:0] scores_next [0:K-1]; + + // ── Static output assignments ────────────────────────────────────────── + assign candidate_ready_o = 1'b1; + assign update_pending_o = 1'b0; + + // ── Combinational: insertion logic ───────────────────────────────────── + always_comb begin + int insert_idx; + insert_idx = K; // always assigned — default: no insertion + + // Default: retain current state + for (int i = 0; i < K; i++) begin + rows_next[i] = rows_q[i]; + scores_next[i] = scores_q[i]; + end + + // Only evaluate insertion when a valid candidate is presented + if (candidate_valid_i) begin + // Find the first slot where candidate is strictly better than the + // current occupant. "Better" means higher score, or equal score + // with lower row (tie-break). + // Note: guard on insert_idx == K avoids loop-local break which + // Yosys rejects; the effect is the same — only the first match + // is captured because subsequent iterations see insert_idx != K. + for (int i = 0; i < K; i++) begin + if (insert_idx == K && + (candidate_score_i > scores_q[i] || + (candidate_score_i == scores_q[i] && candidate_row_i < rows_q[i]))) begin + insert_idx = i; + end + end + + // Shift slots down from the insertion point and insert. + // Static loop bounds with guard for Yosys compatibility. + if (insert_idx < K) begin + for (int i = 0; i < K; i++) begin + if (i == insert_idx) begin + rows_next[i] = candidate_row_i; + scores_next[i] = candidate_score_i; + end else if (i > insert_idx) begin + rows_next[i] = rows_q[i - 1]; + scores_next[i] = scores_q[i - 1]; + end + // i < insert_idx: keep default from copy loop above + end + end + end + end + + // ── Sequential: state update ─────────────────────────────────────────── + always_ff @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + for (int i = 0; i < K; i++) begin + rows_q[i] <= {ROW_BITS{1'b1}}; // all-ones sentinel + scores_q[i] <= '0; + end + end else if (clear_i) begin + for (int i = 0; i < K; i++) begin + rows_q[i] <= {ROW_BITS{1'b1}}; + scores_q[i] <= '0; + end + end else if (candidate_valid_i) begin + for (int i = 0; i < K; i++) begin + rows_q[i] <= rows_next[i]; + scores_q[i] <= scores_next[i]; + end + end + end + + // ── Combinational: pack arrays into flat outputs ─────────────────────── + always_comb begin + topk_rows_o = '0; + topk_scores_o = '0; + for (int i = 0; i < K; i++) begin + topk_rows_o[i*ROW_BITS +: ROW_BITS] = rows_q[i]; + topk_scores_o[i*SCORE_BITS +: SCORE_BITS] = scores_q[i]; + end + end + +endmodule diff --git a/hw/sim/Makefile b/hw/sim/Makefile index 9e7be1d..8ee4b56 100644 --- a/hw/sim/Makefile +++ b/hw/sim/Makefile @@ -1,5 +1,5 @@ PYTHON ?= python -MODULE_TESTS := cam_core_banked match_engine_pipeline cam_write_noise cam_read_noise popcount_pipeline +MODULE_TESTS := cam_core_banked candidate_fifo match_engine_pipeline cam_write_noise cam_read_noise popcount_pipeline topk_tracker result_serializer .PHONY: help test-all test-top test-modules test-module test-model test-perf clean $(MODULE_TESTS:%=test-module-%) diff --git a/hw/sim/mk/rtl-sources.mk b/hw/sim/mk/rtl-sources.mk index b38827c..37211f1 100644 --- a/hw/sim/mk/rtl-sources.mk +++ b/hw/sim/mk/rtl-sources.mk @@ -11,6 +11,9 @@ RTL_READ_NOISE := $(RTL_NOISE_MASK) $(RTL_RANDOM) $(RTL_ROOT)/noise/cam_read_noi RTL_CAM_CORE_BANKED := $(RTL_ROOT)/core/cam_core_banked.sv RTL_MATCH_ENGINE := \ $(RTL_ROOT)/core/popcount_pipeline.sv \ + $(RTL_ROOT)/core/candidate_fifo.sv \ + $(RTL_ROOT)/core/topk_tracker.sv \ + $(RTL_ROOT)/core/result_serializer.sv \ $(RTL_ROOT)/core/match_engine_pipeline.sv RTL_CAM_NOISY := \ diff --git a/hw/sim/tests/modules/candidate_fifo/Makefile b/hw/sim/tests/modules/candidate_fifo/Makefile new file mode 100644 index 0000000..0fabaa4 --- /dev/null +++ b/hw/sim/tests/modules/candidate_fifo/Makefile @@ -0,0 +1,32 @@ +SIM_ROOT := $(abspath ../../..) +RTL_ROOT := $(abspath $(SIM_ROOT)/../rtl) +include $(SIM_ROOT)/mk/rtl-sources.mk + +TOPLEVEL := candidate_fifo +COCOTB_TEST_MODULES := tests.modules.candidate_fifo.test_candidate_fifo +VERILOG_SOURCES := $(RTL_ROOT)/core/candidate_fifo.sv + +# Allow override from command line (e.g. make FIFO_DEPTH=10) +FIFO_DEPTH ?= 16 +EXTRA_ARGS += +define+FIFO_DEPTH=$(FIFO_DEPTH) + +# Parameter-specific build directory so depth variants never collide. +SIM_BUILD := sim_build/fifo_depth_$(FIFO_DEPTH) + +include $(SIM_ROOT)/mk/cocotb-common.mk + +# ── Depth-specific test targets ────────────────────────────────────────── +# Recompile with a non-default DEPTH and filter to the relevant test(s). +.PHONY: test-depth-10 test-depth-1 + +test-depth-10: + $(MAKE) -B -f Makefile results.xml FIFO_DEPTH=10 \ + COCOTB_TEST_FILTER=non_power_of_two + +test-depth-1: + $(MAKE) -B -f Makefile results.xml FIFO_DEPTH=1 \ + COCOTB_TEST_FILTER=depth_one + +# Also remove variant build directories on clean. +clean:: + rm -rf sim_build/fifo_depth_10 sim_build/fifo_depth_1 diff --git a/hw/sim/tests/modules/candidate_fifo/__init__.py b/hw/sim/tests/modules/candidate_fifo/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hw/sim/tests/modules/candidate_fifo/test_candidate_fifo.py b/hw/sim/tests/modules/candidate_fifo/test_candidate_fifo.py new file mode 100644 index 0000000..e79d3c9 --- /dev/null +++ b/hw/sim/tests/modules/candidate_fifo/test_candidate_fifo.py @@ -0,0 +1,306 @@ +from __future__ import annotations + +import cocotb +from cocotb.clock import Clock +from cocotb.triggers import RisingEdge + + +# ── Helpers ────────────────────────────────────────────────────────────────── + + +async def reset_fifo(dut): + """Assert reset for 5 cycles, release, then wait 2 cycles.""" + dut.rst_n.value = 0 + dut.wr_valid_i.value = 0 + dut.rd_ready_i.value = 0 + dut.wr_row_i.value = 0 + dut.wr_score_i.value = 0 + for _ in range(5): + await RisingEdge(dut.clk) + dut.rst_n.value = 1 + for _ in range(2): + await RisingEdge(dut.clk) + + +async def push(dut, row: int, score: int): + """Push a (row, score) entry, waiting until wr_ready_o is asserted.""" + while not dut.wr_ready_o.value: + await RisingEdge(dut.clk) + dut.wr_row_i.value = row + dut.wr_score_i.value = score + dut.wr_valid_i.value = 1 + await RisingEdge(dut.clk) + dut.wr_valid_i.value = 0 + + +async def pop(dut) -> tuple[int, int]: + """Pop an entry, waiting until rd_valid_o is asserted. Return (row, score).""" + while not dut.rd_valid_o.value: + await RisingEdge(dut.clk) + # Sample data in the cycle rd_valid_o is high, before accepting + row = int(dut.rd_row_o.value) + score = int(dut.rd_score_o.value) + dut.rd_ready_i.value = 1 + await RisingEdge(dut.clk) + dut.rd_ready_i.value = 0 + return (row, score) + + +async def push_pop_simultaneous(dut, row: int, score: int) -> tuple[int, int]: + """Push and pop in the same cycle. Return the popped (row, score).""" + # Capture read data before accepting (combinational read) + row_out = int(dut.rd_row_o.value) + score_out = int(dut.rd_score_o.value) + # Assert both handshakes + dut.wr_row_i.value = row + dut.wr_score_i.value = score + dut.wr_valid_i.value = 1 + dut.rd_ready_i.value = 1 + await RisingEdge(dut.clk) + dut.wr_valid_i.value = 0 + dut.rd_ready_i.value = 0 + return (row_out, score_out) + + +# ── Test 1: Reset ─────────────────────────────────────────────────────────── + + +@cocotb.test() +async def reset_asserts_empty_and_defaults(dut): + """ + After reset: + empty_o == 1 + full_o == 0 + rd_valid_o == 0 + wr_ready_o == 1 + """ + cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) + await reset_fifo(dut) + + assert dut.empty_o.value == 1, "empty_o should be 1 after reset" + assert dut.full_o.value == 0, "full_o should be 0 after reset" + assert dut.rd_valid_o.value == 0, "rd_valid_o should be 0 after reset" + assert dut.wr_ready_o.value == 1, "wr_ready_o should be 1 after reset" + + +# ── Test 2: Order preservation ────────────────────────────────────────────── + + +@cocotb.test() +async def fifo_preserves_order(dut): + """ + Push [(3,44),(1,55),(7,22),(2,55)] then pop and verify each entry appears + in the same order. After draining, empty_o == 1. + """ + cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) + await reset_fifo(dut) + + entries = [(3, 44), (1, 55), (7, 22), (2, 55)] + + for row, score in entries: + await push(dut, row, score) + + for expected_row, expected_score in entries: + row, score = await pop(dut) + assert row == expected_row, ( + f"row mismatch: got {row}, expected {expected_row}" + ) + assert score == expected_score, ( + f"score mismatch: got {score}, expected {expected_score}" + ) + + assert dut.empty_o.value == 1, "fifo should be empty after draining all entries" + + +# ── Test 3: Full / back-pressure / drain ──────────────────────────────────── + + +@cocotb.test() +async def fifo_full_backpressure_and_drain(dut): + """ + Fill the FIFO until full (discover depth), confirm backpressure, + pop one, push a new item, drain all, verify final inserted item + (99, 999) is preserved. + """ + cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) + await reset_fifo(dut) + + # ── Fill the FIFO until full ── + depth = 0 + while not dut.full_o.value: + await push(dut, depth, depth * 10) + depth += 1 + + assert depth > 0, "should have pushed at least one item" + assert dut.full_o.value == 1, "fifo should report full after DEPTH pushes" + assert dut.wr_ready_o.value == 0, "wr_ready should be 0 when full" + + # ── Pop one ── + row, score = await pop(dut) + assert row == 0 and score == 0, ( + f"first popped entry should be (0,0), got ({row},{score})" + ) + + assert dut.full_o.value == 0, "fifo should no longer be full after one pop" + assert dut.wr_ready_o.value == 1, "wr_ready should be 1 after one pop" + + # ── Push a new item ── + await push(dut, 99, 999) + + # ── Drain all entries ── + # Expected order: (1,10), (2,20), … (depth-1, (depth-1)*10), (99,999) + for i in range(1, depth): + row, score = await pop(dut) + assert row == i, ( + f"row mismatch during drain: got {row}, expected {i}" + ) + assert score == i * 10, ( + f"score mismatch during drain: got {score}, expected {i * 10}" + ) + + row, score = await pop(dut) + assert row == 99 and score == 999, ( + f"final inserted item wrong: got ({row},{score}), expected (99,999)" + ) + + assert dut.empty_o.value == 1, "fifo should be empty after draining everything" + + +# ── Test 4: Simultaneous push+pop at full ─────────────────────────────────── + + +@cocotb.test() +async def test_simultaneous_push_pop_when_full(dut): + """ + When the FIFO is full, a simultaneous read should allow a write to pass + through in the same cycle (wr_ready_o goes high combinationally because + of the read handshake). Verify the pop returns the oldest item, the + push inserts a new item, and the fill level stays at depth. + """ + cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) + await reset_fifo(dut) + + # ── Fill to full ── + depth = 0 + while not dut.full_o.value: + await push(dut, depth, depth * 10) + depth += 1 + + assert dut.full_o.value == 1 + assert dut.wr_ready_o.value == 0 # backpressure active + + # ── Simultaneous push+pop ── + # wr_ready_o should be 1 this cycle because rd_valid_o && rd_ready_i + old_row, old_score = await push_pop_simultaneous(dut, 99, 999) + assert old_row == 0 and old_score == 0, ( + f"simultaneous pop should return the oldest item, got ({old_row},{old_score})" + ) + + # Fill level unchanged (pop frees a slot, push immediately refills it) + assert dut.full_o.value == 1, "should remain full after simultaneous push+pop" + + # ── Drain — expected: (1,10) … (depth-1,…), (99,999) ── + for i in range(1, depth): + row, score = await pop(dut) + assert row == i, ( + f"row mismatch: got {row}, expected {i}" + ) + + row, score = await pop(dut) + assert row == 99 and score == 999, ( + f"final item wrong: got ({row},{score})" + ) + + assert dut.empty_o.value == 1 + + +# ── Test 5: Non-power-of-two depth ────────────────────────────────────────── + + +@cocotb.test() +async def test_non_power_of_two_depth(dut): + """ + Validate pointer wrap at a non-power-of-two DEPTH boundary (e.g. 10). + Fill the FIFO, then perform multiple pop+push cycles to force the write + pointer past a binary-wrap boundary, then drain and verify data integrity. + """ + cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) + await reset_fifo(dut) + + # ── Fill to full ── + depth = 0 + while not dut.full_o.value: + await push(dut, depth, depth) + depth += 1 + + # ── Force several pointer wraps by pop+push ── + # For a broken RTL that wraps at 2**clog2(DEPTH) instead of DEPTH, + # enough iterations will hit out-of-bounds storage or corrupt data. + wraps = depth * 3 # well past the binary wrap point + for i in range(wraps): + row, score = await pop(dut) + await push(dut, 500 + i, 600 + i) + + # ── Drain and verify count ── + items = [] + while dut.rd_valid_o.value: + row, score = await pop(dut) + items.append((row, score)) + + assert len(items) == depth, ( + f"expected {depth} items after {wraps} push/pop cycles, got {len(items)}" + ) + + # All items should be from the latest wrap cycle (order preserved) + # Final batch started at iteration wraps - depth: + for idx, (row, _score) in enumerate(items): + expected_row = 500 + (wraps - depth + idx) + assert row == expected_row, ( + f"item {idx}: expected row {expected_row}, got {row}" + ) + + assert dut.empty_o.value == 1 + + +# ── Test 6: Depth = 1 ─────────────────────────────────────────────────────── + + +@cocotb.test() +async def test_depth_one(dut): + """ + Validate DEPTH=1 FIFO: push one item → full, pop → empty, + push another → full, pop → empty. + + Gated: pushes one item; if the FIFO does *not* become full, we are + compiled with DEPTH > 1 and the test silently passes (skips). + (Use ``make test-depth-1`` to run with FIFO_DEPTH=1.) + """ + cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) + await reset_fifo(dut) + + # ── Probe: is DEPTH == 1? ──────────────────────────────────────────── + await push(dut, 42, 100) + if not dut.full_o.value: + # Not compiled at DEPTH=1; pop the item back and return. + await pop(dut) + return + + # ── Full DEPTH=1 logic follows ─────────────────────────────────────── + assert dut.wr_ready_o.value == 0, "wr_ready should be 0 when full" + + row, score = await pop(dut) + assert row == 42 and score == 100, ( + f"popped ({row},{score}), expected (42,100)" + ) + assert dut.empty_o.value == 1, "depth-1 FIFO should be empty after one pop" + assert dut.full_o.value == 0 + + await push(dut, 7, 55) + assert dut.full_o.value == 1 + assert dut.empty_o.value == 0 + + row, score = await pop(dut) + assert row == 7 and score == 55 + + assert dut.empty_o.value == 1 + assert dut.full_o.value == 0 diff --git a/hw/sim/tests/modules/match_engine_pipeline/Makefile b/hw/sim/tests/modules/match_engine_pipeline/Makefile index 8509d40..57e08f3 100644 --- a/hw/sim/tests/modules/match_engine_pipeline/Makefile +++ b/hw/sim/tests/modules/match_engine_pipeline/Makefile @@ -6,4 +6,24 @@ TOPLEVEL := match_engine_pipeline COCOTB_TEST_MODULES := tests.modules.match_engine_pipeline.test_match_engine_pipeline VERILOG_SOURCES := $(RTL_MATCH_ENGINE) +# Allow override from command line (e.g. make TOPK_K=1) +TOPK_K ?= 4 +EXTRA_ARGS += +define+TOPK_K=$(TOPK_K) +export TOPK_K + +# Parameter-specific build directory so K variants never collide. +SIM_BUILD := sim_build/k_$(TOPK_K) + include $(SIM_ROOT)/mk/cocotb-common.mk + +# ── K-specific test targets ────────────────────────────────────────────── +# Recompile with K=1 and filter to the K=1 test. +.PHONY: test-k1 + +test-k1: + $(MAKE) -B -f Makefile results.xml TOPK_K=1 \ + COCOTB_TEST_FILTER=match_engine_topk_k1 + +# Also remove all module-local build artifacts, including legacy and variant dirs. +clean:: + rm -rf sim_build diff --git a/hw/sim/tests/modules/match_engine_pipeline/test_match_engine_pipeline.py b/hw/sim/tests/modules/match_engine_pipeline/test_match_engine_pipeline.py index 909e330..c9a8837 100644 --- a/hw/sim/tests/modules/match_engine_pipeline/test_match_engine_pipeline.py +++ b/hw/sim/tests/modules/match_engine_pipeline/test_match_engine_pipeline.py @@ -1,8 +1,13 @@ from __future__ import annotations +import os + import cocotb from cocotb.clock import Clock -from cocotb.triggers import RisingEdge +from cocotb.triggers import RisingEdge, Timer + +# Read TOPK_K from environment (set by Makefile); default 4 for K=4-only tests. +TOPK_K = int(os.environ.get("TOPK_K", "4")) async def reset_match(dut): @@ -21,8 +26,99 @@ async def reset_match(dut): await RisingEdge(dut.clk) +def score_for(query: int, value: int, width: int) -> int: + """Compute popcount of ~(query ^ value), i.e. number of matching bits.""" + return width - ((query ^ value) & ((1 << width) - 1)).bit_count() + + +def golden_topk( + rows_and_hashes: list[tuple[int, int]], query: int, width: int, k: int +) -> list[tuple[int, int]]: + """Return the Top-K (row, score) pairs sorted by score desc, row asc.""" + scored = [(row, score_for(query, value, width)) + for row, value in rows_and_hashes] + return sorted(scored, key=lambda item: (-item[1], item[0]))[:k] + + +async def collect_serial_results( + dut, k: int, timeout_cycles: int = 200 +) -> list[tuple[int, int, int, int]]: + """Collect serialised Top-K results from the pipeline. + + Returns list of (rank, row, score, last) tuples. + Asserts timeout if the stream does not complete. + """ + dut.result_ready.value = 1 + observed: list[tuple[int, int, int, int]] = [] + for _ in range(timeout_cycles): + if int(dut.result_valid.value): + rank = int(dut.result_rank.value) if hasattr( + dut, "result_rank") else len(observed) + row = int(dut.result_row.value) + score = int(dut.result_score.value) + last = int(dut.result_last.value) if hasattr( + dut, "result_last") else int(len(observed) == k - 1) + observed.append((rank, row, score, last)) + if last: + return observed + await RisingEdge(dut.clk) + raise AssertionError( + f"result stream did not complete after {timeout_cycles} cycles" + ) + + +async def feed_all_batches( + dut, + rows_and_hashes: list[tuple[int, int]] | None = None, + lane_mask: int | None = None, +): + """Feed all batches to the pipeline. + + If *rows_and_hashes* is provided, the test uses those values for each row + (row_id -> hash) instead of the default hash=query (with special rows). + If *lane_mask* is provided, only those lanes are marked valid in each batch. + """ + LANES = len(dut.rd_lane_valid_i) + ROW_BITS = len(dut.rd_row_ids_i) // LANES + HASH_BITS = len(dut.rd_hashes_i) // LANES + NUM_ROWS = 1 << len(dut.rd_base_row_o) + + for base in range(0, NUM_ROWS, LANES): + while int(dut.rd_valid_o.value) == 0: + await RisingEdge(dut.clk) + assert int(dut.rd_base_row_o.value) == base + rows = 0 + hashes = 0 + for lane in range(LANES): + row_id = base + lane + rows |= row_id << (lane * ROW_BITS) + if rows_and_hashes is not None and row_id in rows_and_hashes: + row_hash = rows_and_hashes[row_id] + else: + row_hash = 0 + hashes |= row_hash << (lane * HASH_BITS) + dut.rd_row_ids_i.value = rows + dut.rd_hashes_i.value = hashes + if lane_mask is not None: + dut.rd_lane_valid_i.value = lane_mask + else: + dut.rd_lane_valid_i.value = (1 << LANES) - 1 + dut.rd_valid_i.value = 1 + await RisingEdge(dut.clk) + dut.rd_valid_i.value = 0 + + +# ============================================================================== +# Test: Top-1 smoke +# ============================================================================== + + @cocotb.test() async def match_engine_returns_top1_after_pipeline_drain(dut): + """Top-1 smoke test — verify basic result_valid/result_row/result_score. + + Works with both old Top-1 and new Top-K serial interfaces. + """ cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) await reset_match(dut) @@ -38,23 +134,8 @@ async def match_engine_returns_top1_after_pipeline_drain(dut): await RisingEdge(dut.clk) dut.query_valid.value = 0 - for base in range(0, NUM_ROWS, LANES): - while int(dut.rd_valid_o.value) == 0: - await RisingEdge(dut.clk) - assert int(dut.rd_base_row_o.value) == base - rows = 0 - hashes = 0 - for lane in range(LANES): - row_id = base + lane - rows |= row_id << (lane * ROW_BITS) - row_hash = query if row_id == TARGET_ROW else 0 - hashes |= row_hash << (lane * HASH_BITS) - dut.rd_row_ids_i.value = rows - dut.rd_hashes_i.value = hashes - dut.rd_lane_valid_i.value = (1 << LANES) - 1 - dut.rd_valid_i.value = 1 - await RisingEdge(dut.clk) - dut.rd_valid_i.value = 0 + rows_and_hashes: dict[int, int] = {9: query} + await feed_all_batches(dut, rows_and_hashes) for _ in range(20): if int(dut.result_valid.value): @@ -64,3 +145,417 @@ async def match_engine_returns_top1_after_pipeline_drain(dut): assert int(dut.result_valid.value) == 1 assert int(dut.result_row.value) == TARGET_ROW assert int(dut.result_score.value) == HASH_BITS + + +# ============================================================================== +# Test: Top-K all-valid +# ============================================================================== + + +@cocotb.test() +async def match_engine_keeps_multiple_topk_candidates_from_same_batch(dut): + """Verify Top-K pipeline returns K best matches in rank order. + + Feed all rows with known hash values so the golden Top-K is predictable. + """ + if TOPK_K != 4: + dut._log.info(f"Skipping: hardcoded K=4 test (TOPK_K={TOPK_K})") + return + cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) + await reset_match(dut) + + LANES = len(dut.rd_lane_valid_i) + ROW_BITS = len(dut.rd_row_ids_i) // LANES + HASH_BITS = len(dut.rd_hashes_i) // LANES + NUM_ROWS = 1 << len(dut.rd_base_row_o) + K = 4 + + query = (1 << HASH_BITS) - 1 + special = {0: query, 1: query ^ 1, 2: query ^ 3, 3: query ^ 7} + + rows_and_hashes: list[tuple[int, int]] = [] + dut.query_hash.value = query + dut.query_valid.value = 1 + await RisingEdge(dut.clk) + dut.query_valid.value = 0 + + for base in range(0, NUM_ROWS, LANES): + while int(dut.rd_valid_o.value) == 0: + await RisingEdge(dut.clk) + rows = 0 + hashes = 0 + for lane in range(LANES): + row_id = base + lane + row_hash = special.get(row_id, 0) + rows_and_hashes.append((row_id, row_hash)) + rows |= row_id << (lane * ROW_BITS) + hashes |= row_hash << (lane * HASH_BITS) + dut.rd_row_ids_i.value = rows + dut.rd_hashes_i.value = hashes + dut.rd_lane_valid_i.value = (1 << LANES) - 1 + dut.rd_valid_i.value = 1 + await RisingEdge(dut.clk) + dut.rd_valid_i.value = 0 + + expected = golden_topk(rows_and_hashes, query, HASH_BITS, K) + observed = await collect_serial_results(dut, K, timeout_cycles=800) + + assert [(row, score) for _, row, score, _ in observed] == expected, ( + f"observed={[(r, s) for _, r, s, _ in observed]} != expected={expected}" + ) + assert [rank for rank, _, _, _ in observed] == list(range(K)), ( + "ranks are not sequential" + ) + assert observed[-1][3] == 1, "last flag not set on final result" + + +# ============================================================================== +# Test: Invalid-mask coverage — lane 0 invalid, all-invalid batch +# ============================================================================== + + +@cocotb.test() +async def match_engine_handles_invalid_lane_masks(dut): + """Pipeline completes correctly with invalid lanes and all-invalid batches. + + Coverage: + - Batch 0 is driven with rd_lane_valid_i=0 (all-invalid) + → batch_scores_done vacuously true, zero candidates pushed. + - All other batches use lane_mask = 0b0110 (lanes 1 and 2 only). + - Invalid lanes (0, 3-7) carry high-score trap hashes that WOULD enter + the Top-K if the pipeline wrongly pushed them. + - Golden Top-K is computed from valid-lane rows only and compared + against the full serialised output — not just a subset check. + """ + if TOPK_K != 4: + dut._log.info(f"Skipping: hardcoded K=4 test (TOPK_K={TOPK_K})") + return + cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) + await reset_match(dut) + + LANES = len(dut.rd_lane_valid_i) + ROW_BITS = len(dut.rd_row_ids_i) // LANES + HASH_BITS = len(dut.rd_hashes_i) // LANES + NUM_ROWS = 1 << len(dut.rd_base_row_o) + K = 4 + + query = (1 << HASH_BITS) - 1 + trap_hash = query # perfect score; would be Top-1 if wrongly pushed + + # Only rows on valid lanes (1, 2) from batches AFTER the all-invalid one + # contribute candidates. Use distinct scores so the golden ordering + # is unambiguous. + valid_rows: dict[int, int] = {9: query, 10: query ^ 1} + + # Track hash values passed per row (including trap rows on invalid lanes) + # for golden-topk computation. + valid_rows_and_hashes: list[tuple[int, int]] = [] + + dut.query_hash.value = query + dut.query_valid.value = 1 + await RisingEdge(dut.clk) + dut.query_valid.value = 0 + + for base in range(0, NUM_ROWS, LANES): + while int(dut.rd_valid_o.value) == 0: + await RisingEdge(dut.clk) + # Batch 0 is all-invalid; all others use lanes 1,2 only. + cur_mask = 0x00 if base == 0 else 0b0110 + + rows = 0 + hashes = 0 + for lane in range(LANES): + row_id = base + lane + rows |= row_id << (lane * ROW_BITS) + # On invalid lanes drive a trap hash (perfect score). + # On valid lanes drive the designated hash (or 0 if not special). + if cur_mask & (1 << lane): + row_hash = valid_rows.get(row_id, 0) + valid_rows_and_hashes.append((row_id, row_hash)) + else: + row_hash = trap_hash # would be Top-1 if wrongly pushed + hashes |= row_hash << (lane * HASH_BITS) + dut.rd_row_ids_i.value = rows + dut.rd_hashes_i.value = hashes + dut.rd_lane_valid_i.value = cur_mask + dut.rd_valid_i.value = 1 + await RisingEdge(dut.clk) + dut.rd_valid_i.value = 0 + + expected = golden_topk(valid_rows_and_hashes, query, HASH_BITS, K) + observed = await collect_serial_results(dut, K, timeout_cycles=800) + + assert [(row, score) for _, row, score, _ in observed] == expected, ( + f"observed={[(r, s) for _, r, s, _ in observed]} != expected={expected}" + ) + assert [rank for rank, _, _, _ in observed] == list(range(K)), ( + "ranks are not sequential" + ) + assert observed[-1][3] == 1, "last flag not set on final result" + + +# ============================================================================== +# Test: result_ready stall stability +# ============================================================================== + + +@cocotb.test() +async def match_engine_result_ready_stall_preserves_output(dut): + """When result_ready is deasserted while result_valid is high, the output + signals (result_rank, result_row, result_score, result_last) must remain + stable until result_ready is re-asserted. + """ + if TOPK_K != 4: + dut._log.info(f"Skipping: hardcoded K=4 test (TOPK_K={TOPK_K})") + return + cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) + await reset_match(dut) + + LANES = len(dut.rd_lane_valid_i) + ROW_BITS = len(dut.rd_row_ids_i) // LANES + HASH_BITS = len(dut.rd_hashes_i) // LANES + NUM_ROWS = 1 << len(dut.rd_base_row_o) + K = 4 + + query = (1 << HASH_BITS) - 1 + special = {0: query, 1: query ^ 1, 2: query ^ 3, 3: query ^ 7} + + rows_and_hashes: list[tuple[int, int]] = [] + dut.query_hash.value = query + dut.query_valid.value = 1 + await RisingEdge(dut.clk) + dut.query_valid.value = 0 + + await feed_all_batches(dut) + + # Wait for first result + for _ in range(200): + await RisingEdge(dut.clk) + if int(dut.result_valid.value): + break + + assert int(dut.result_valid.value) == 1, "result_valid never went high" + + # Capture the first-beat values + rank0 = int(dut.result_rank.value) + row0 = int(dut.result_row.value) + score0 = int(dut.result_score.value) + last0 = int(dut.result_last.value) + + # Stall — deassert result_ready for several cycles + dut.result_ready.value = 0 + for _ in range(5): + await RisingEdge(dut.clk) + assert int(dut.result_valid.value) == 1, ( + "result_valid dropped during stall" + ) + assert int(dut.result_rank.value) == rank0 + assert int(dut.result_row.value) == row0 + assert int(dut.result_score.value) == score0 + assert int(dut.result_last.value) == last0 + + # Release stall — should advance to next rank + dut.result_ready.value = 1 + await RisingEdge(dut.clk) + + # Collect the rest + observed: list[tuple[int, int, int, int]] = [(rank0, row0, score0, last0)] + for _ in range(200): + if int(dut.result_valid.value): + rank = int(dut.result_rank.value) + row = int(dut.result_row.value) + score = int(dut.result_score.value) + last = int(dut.result_last.value) + observed.append((rank, row, score, last)) + if last: + break + await RisingEdge(dut.clk) + + assert len(observed) == K, ( + f"expected {K} results, got {len(observed)}" + ) + assert observed[-1][3] == 1, "last flag not set on final result" + + +# ============================================================================== +# Test: Toggled ready with golden comparison +# ============================================================================== + + +@cocotb.test() +async def match_engine_holds_serial_result_under_ready_stalls(dut): + """Toggle result_ready during serialisation and verify golden Top-K matches. + + Uses query=(1< int: + """Pack a list of values into a flattened integer (LSB = index 0).""" + result = 0 + for idx, value in enumerate(values): + result |= value << (idx * width) + return result + + +# ── Helpers ──────────────────────────────────────────────────────────────────── + + +async def reset_serializer(dut): + """Drive reset active-low for 5 cycles, then release; zero all inputs.""" + dut.rst_n.value = 0 + dut.start_i.value = 0 + dut.result_ready_i.value = 0 + dut.topk_rows_i.value = 0 + dut.topk_scores_i.value = 0 + for _ in range(5): + await RisingEdge(dut.clk) + dut.rst_n.value = 1 + for _ in range(2): + await RisingEdge(dut.clk) + + +async def assert_output_stable(dut, rank: int, row: int, score: int, + last: bool, valid: bool): + """Assert output signals match expected values.""" + assert int(dut.result_rank_o.value) == rank, \ + f"rank: expected {rank}, got {int(dut.result_rank_o.value)}" + assert int(dut.result_row_o.value) == row, \ + f"row: expected {row}, got {int(dut.result_row_o.value)}" + assert int(dut.result_score_o.value) == score, \ + f"score: expected {score}, got {int(dut.result_score_o.value)}" + assert bool(dut.result_last_o.value) == last, \ + f"last: expected {last}, got {bool(dut.result_last_o.value)}" + assert bool(dut.result_valid_o.value) == valid, \ + f"valid: expected {valid}, got {bool(dut.result_valid_o.value)}" + + +# ── Test 1: Serializer outputs rank order and last ───────────────────────────── + + +@cocotb.test() +async def serializer_outputs_rank_order_and_last(dut): + """ + Start serialization with K=4 results. Clock through all 4 ranks, verifying + row/score/last at each step. Assert done_o pulses after last rank. + + Gated: requires TOPK_K=4 (the default). Use ``make test-k1`` for K=1. + """ + if K != 4: + cocotb.log.info("Skipping (designed for TOPK_K=4, got %d)", K) + return + + cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) + await reset_serializer(dut) + + # Pre-defined top-4 results + rows = [42, 17, 99, 5] + scores = [10, 9, 8, 7] + + # Drive packed inputs and assert start + dut.topk_rows_i.value = pack(rows, ROW_BITS) + dut.topk_scores_i.value = pack(scores, SCORE_BITS) + dut.start_i.value = 1 + await RisingEdge(dut.clk) + dut.start_i.value = 0 + + # DUT should be busy, valid, showing rank 0 + assert bool(dut.busy_o.value), "busy_o should be high after start" + await assert_output_stable(dut, rank=0, row=rows[0], score=scores[0], + last=False, valid=True) + + # Cycle through remaining ranks with ready held high + for r in range(1, K): + dut.result_ready_i.value = 1 + await RisingEdge(dut.clk) + is_last = (r == K - 1) + await assert_output_stable(dut, rank=r, row=rows[r], score=scores[r], + last=is_last, valid=True) + + # After last rank fire, next cycle should show done_o and deassert busy + await RisingEdge(dut.clk) + assert bool(dut.done_o.value), "done_o should be high after last rank fire" + assert not bool(dut.busy_o.value), "busy_o should be low after serialization done" + assert not bool(dut.result_valid_o.value), \ + "result_valid_o should be low after serialization done" + + # done_o should be single-cycle pulse + await RisingEdge(dut.clk) + assert not bool(dut.done_o.value), "done_o should be single-cycle pulse" + assert int(dut.result_rank_o.value) == 0, \ + "rank should reset to 0 after done" + + +# ── Test 2: Holds output when not ready ──────────────────────────────────────── + + +@cocotb.test() +async def serializer_holds_output_when_not_ready(dut): + """ + After start, with result_ready_i=0, output remains stable. + After ready is asserted, advances to next rank. + + Gated: requires TOPK_K=4 (the default). Use ``make test-k1`` for K=1. + """ + if K != 4: + cocotb.log.info("Skipping (designed for TOPK_K=4, got %d)", K) + return + + cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) + await reset_serializer(dut) + + rows = [10, 20, 30, 40] + scores = [5, 4, 3, 2] + + dut.topk_rows_i.value = pack(rows, ROW_BITS) + dut.topk_scores_i.value = pack(scores, SCORE_BITS) + dut.result_ready_i.value = 0 # not ready + dut.start_i.value = 1 + await RisingEdge(dut.clk) + dut.start_i.value = 0 + + # Output should present rank 0 but hold because ready is low + await assert_output_stable(dut, rank=0, row=rows[0], score=scores[0], + last=False, valid=True) + + # Stall for 3 cycles — mutating inputs to verify they don't leak through + # (snapshot must hold original values captured at start) + for cycle in range(3): + # Corrupt the live inputs — a buggy combinational slice would change + dut.topk_rows_i.value = pack([cycle + 100, 0, 0, 0], ROW_BITS) + dut.topk_scores_i.value = pack([cycle + 50, 0, 0, 0], SCORE_BITS) + await RisingEdge(dut.clk) + await assert_output_stable(dut, rank=0, row=rows[0], score=scores[0], + last=False, valid=True) + + # Restore original inputs and assert ready for 1 cycle; rank advances to 1 + dut.topk_rows_i.value = pack(rows, ROW_BITS) + dut.topk_scores_i.value = pack(scores, SCORE_BITS) + dut.result_ready_i.value = 1 + await RisingEdge(dut.clk) + await assert_output_stable(dut, rank=1, row=rows[1], score=scores[1], + last=False, valid=True) + + # Stall again at rank 1 for 2 cycles — mutate inputs again + dut.result_ready_i.value = 0 + for cycle in range(2): + dut.topk_rows_i.value = pack([cycle + 200, 0, 0, 0], ROW_BITS) + dut.topk_scores_i.value = pack([cycle + 100, 0, 0, 0], SCORE_BITS) + await RisingEdge(dut.clk) + await assert_output_stable(dut, rank=1, row=rows[1], score=scores[1], + last=False, valid=True) + + +# ── Test 3: K=1 edge case ────────────────────────────────────────────────────── + + +@cocotb.test() +async def serializer_handles_k1(dut): + """ + When K=1 (overridden via generics), start and fire produce done_o and + deassert busy on the next cycle. Run via ``make test-k1``. + """ + if K != 1: + cocotb.log.info("Skipping (designed for TOPK_K=1, got %d)", K) + return + + cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) + await reset_serializer(dut) + + rows = [100] + scores = [50] + + dut.topk_rows_i.value = pack(rows, ROW_BITS) + dut.topk_scores_i.value = pack(scores, SCORE_BITS) + dut.start_i.value = 1 + await RisingEdge(dut.clk) + dut.start_i.value = 0 + + # K=1: rank 0 is also the last; valid asserted + assert bool(dut.busy_o.value), "busy_o should be high after start" + await assert_output_stable(dut, rank=0, row=rows[0], score=scores[0], + last=True, valid=True) + + # Fire the one and only result + dut.result_ready_i.value = 1 + await RisingEdge(dut.clk) + + # done_o should pulse, busy deasserted + assert bool(dut.done_o.value), "done_o should pulse after K=1 fire" + assert not bool(dut.busy_o.value), "busy_o should be low after done" + assert not bool(dut.result_valid_o.value), "valid should be low after done" + assert int(dut.result_rank_o.value) == 0, "rank should reset to 0" diff --git a/hw/sim/tests/modules/topk_tracker/Makefile b/hw/sim/tests/modules/topk_tracker/Makefile new file mode 100644 index 0000000..8667f3d --- /dev/null +++ b/hw/sim/tests/modules/topk_tracker/Makefile @@ -0,0 +1,9 @@ +SIM_ROOT := $(abspath ../../..) +RTL_ROOT := $(abspath $(SIM_ROOT)/../rtl) +include $(SIM_ROOT)/mk/rtl-sources.mk + +TOPLEVEL := topk_tracker +COCOTB_TEST_MODULES := tests.modules.topk_tracker.test_topk_tracker +VERILOG_SOURCES := $(RTL_ROOT)/core/topk_tracker.sv + +include $(SIM_ROOT)/mk/cocotb-common.mk diff --git a/hw/sim/tests/modules/topk_tracker/test_topk_tracker.py b/hw/sim/tests/modules/topk_tracker/test_topk_tracker.py new file mode 100644 index 0000000..550ea4e --- /dev/null +++ b/hw/sim/tests/modules/topk_tracker/test_topk_tracker.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import cocotb +from cocotb.clock import Clock +from cocotb.triggers import RisingEdge + +# ── Parameter defaults (matching cam_params.svh with NUM_ROWS=4096, HASH_BITS=512) ── +K = 4 +ROW_BITS = 12 +SCORE_BITS = 10 # $clog2(512 + 1) +ROW_SENTINEL = (1 << ROW_BITS) - 1 # all-ones + + +def golden(candidates, k=K): + """Return top-k candidates sorted by score desc, then row asc (higher score wins; lower row breaks ties).""" + return sorted(candidates, key=lambda item: (-item[1], item[0]))[:k] + + +def unpack(dut, k=K): + """Read packed topk_rows_o and topk_scores_o; return list of (row, score) tuples ordered by rank.""" + rows_packed = int(dut.topk_rows_o.value) + scores_packed = int(dut.topk_scores_o.value) + result = [] + for i in range(k): + row = (rows_packed >> (i * ROW_BITS)) & ((1 << ROW_BITS) - 1) + score = (scores_packed >> (i * SCORE_BITS)) & ((1 << SCORE_BITS) - 1) + result.append((row, score)) + return result + + +# ── Helpers ────────────────────────────────────────────────────────────────────── + + +async def reset_tracker(dut): + """Assert reset for 5 cycles, release, then wait 2 cycles.""" + dut.rst_n.value = 0 + dut.clear_i.value = 0 + dut.candidate_valid_i.value = 0 + for _ in range(5): + await RisingEdge(dut.clk) + dut.rst_n.value = 1 + for _ in range(2): + await RisingEdge(dut.clk) + + +async def push_candidate(dut, row: int, score: int): + """Drive one candidate handshake (candidate_ready_o is always 1).""" + dut.candidate_valid_i.value = 1 + dut.candidate_row_i.value = row + dut.candidate_score_i.value = score + await RisingEdge(dut.clk) + dut.candidate_valid_i.value = 0 + + +# ── Test 1: Basic ordering ─────────────────────────────────────────────────────── + + +@cocotb.test() +async def tracker_orders_by_score_then_lower_row(dut): + """ + Push candidates [(5,10), (3,11), (2,11), (9,7), (1,11), (0,3)] and verify + the tracker's Top-4 matches golden: [(1,11), (2,11), (3,11), (5,10)]. + """ + cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) + await reset_tracker(dut) + + candidates = [(5, 10), (3, 11), (2, 11), (9, 7), (1, 11), (0, 3)] + + for row, score in candidates: + await push_candidate(dut, row, score) + + result = unpack(dut) + expected = golden(candidates) + assert result == expected, f"Expected {expected}, got {result}" + assert dut.update_pending_o.value == 0, "update_pending_o should be 0" + + +# ── Test 2: Clear restarts query ───────────────────────────────────────────────── + + +@cocotb.test() +async def tracker_clear_starts_new_query(dut): + """ + After candidate (7,60), pulse clear_i, send candidate (4,12). + Expect rank 0 = (4,12), remaining slots = sentinel rows, score 0. + """ + cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) + await reset_tracker(dut) + + # Push (7, 60) + await push_candidate(dut, 7, 60) + + # Pulse clear + dut.clear_i.value = 1 + await RisingEdge(dut.clk) + dut.clear_i.value = 0 + + # Push (4, 12) + await push_candidate(dut, 4, 12) + + result = unpack(dut) + expected = [(4, 12)] + [(ROW_SENTINEL, 0)] * (K - 1) + assert result == expected, f"Expected {expected}, got {result}" + + +# ── Test 3: Random validation against golden ───────────────────────────────────── + + +@cocotb.test() +async def tracker_matches_random_python_golden(dut): + """ + Push 100 candidates (row in [0,63], score in [0,127]) seeded at 42. + Verify final Top-4 matches Python golden model. + """ + import random + random.seed(42) + + cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) + await reset_tracker(dut) + + max_row = 64 + max_score = 128 + candidates = [] + + for _ in range(100): + row = random.randint(0, max_row - 1) + score = random.randint(0, max_score - 1) + candidates.append((row, score)) + await push_candidate(dut, row, score) + + result = unpack(dut) + expected = golden(candidates) + assert result == expected, f"Expected {expected}, got {result}" diff --git a/hw/sim/tests/top/test_cam_basic.py b/hw/sim/tests/top/test_cam_basic.py index 80ac3ec..ba2148e 100644 --- a/hw/sim/tests/top/test_cam_basic.py +++ b/hw/sim/tests/top/test_cam_basic.py @@ -64,6 +64,63 @@ def dut_score_bits(dut): # ── Helpers ────────────────────────────────────────────────────────────────── +async def collect_topk(dut, timeout_cycles: int = 2000): + """Collect all serial Top-K result beats with result_ready held high. + + Returns list of (rank, row, score, last) tuples. + Raises AssertionError if the stream does not complete within timeout. + """ + dut.result_ready.value = 1 + beats = [] + for _ in range(timeout_cycles): + if int(dut.result_valid.value): + rank = int(dut.result_rank.value) + row = int(dut.result_row.value) + score = int(dut.result_score.value) + last = int(dut.result_last.value) + beats.append((rank, row, score, last)) + if last: + return beats + await RisingEdge(dut.clk) + raise AssertionError("Top-K result stream did not finish") + + +async def query_topk_once(dut, query, timeout_cycles=2000): + """Issue a query, collect full serial Top-K stream, and return beats + Top-1 metadata. + + Returns (beats, top1_index, top1_score, score_debug). + After this call the full result stream has been consumed and the DUT is idle. + """ + await wait_idle(dut) + + dut.query_hash.value = int(query) + dut.query_valid.value = 1 + + # Wait for handshake + while True: + await RisingEdge(dut.clk) + if int(dut.query_ready.value): + break + + dut.query_valid.value = 0 + + # Consume full serial result stream + beats = await collect_topk(dut, timeout_cycles=timeout_cycles) + + # score_debug is available after query completes + num_rows = dut_num_rows(dut) + score_bits = dut_score_bits(dut) + score_debug = None + if hasattr(dut, "score_debug_flat"): + score_debug = unpack_score_debug_flat( + int(dut.score_debug_flat.value), + num_rows, + score_bits, + ) + + return beats, beats[0][1], beats[0][2], score_debug + + async def reset_dut(dut): """Reset the DUT with new handshake interface.""" dut.rst_n.value = 0 @@ -116,41 +173,11 @@ async def write_rows(dut, rows): async def query_once(dut, query): - """Issue a query and return (top1_index, top1_score, score_debug).""" - await wait_idle(dut) - - dut.query_hash.value = int(query) - dut.query_valid.value = 1 - - # Wait for handshake - while True: - await RisingEdge(dut.clk) - if int(dut.query_ready.value): - break - - dut.query_valid.value = 0 - - # Wait for result - while int(dut.result_valid.value) == 0: - await RisingEdge(dut.clk) - - top1_index = int(dut.top1_index.value) - top1_score = int(dut.top1_score.value) - - num_rows = dut_num_rows(dut) - score_bits = dut_score_bits(dut) - score_debug = None - if hasattr(dut, "score_debug_flat"): - score_debug = unpack_score_debug_flat( - int(dut.score_debug_flat.value), - num_rows, - score_bits, - ) - - dut.result_ready.value = 1 - await RisingEdge(dut.clk) - dut.result_ready.value = 0 + """Issue a query and return (top1_index, top1_score, score_debug). + Consumes the full serial Top-K stream and returns rank-0 data. + """ + _, top1_index, top1_score, score_debug = await query_topk_once(dut, query) return top1_index, top1_score, score_debug @@ -187,7 +214,7 @@ async def baseline_no_noise(dut): query = rows[query_index] await write_rows(dut, rows) - top1_index, top1_score, score_debug = await query_once(dut, query) + beats, top1_index, top1_score, score_debug = await query_topk_once(dut, query) expected = match_top1(query, rows, width=hash_bits) @@ -196,6 +223,16 @@ async def baseline_no_noise(dut): assert top1_index == query_index assert top1_score == hash_bits + # Serial Top-K stream verification: beats from the single query above + assert beats[0][0] == 0, "First beat must have rank 0" + assert beats[-1][3] == 1, "Last beat must assert result_last" + # Verify top1 aliases match first beat after stream fully consumed + assert int(dut.top1_index.value) == beats[0][1] + assert int(dut.top1_score.value) == beats[0][2] + # Verify returned top1 matches first beat rank0 + assert top1_index == beats[0][1] + assert top1_score == beats[0][2] + if score_debug is not None: assert np.array_equal(score_debug, expected.scores) @@ -564,10 +601,10 @@ async def query_scan_blocks_writes_until_result_consumed(dut): assert int(dut.wr_ready.value) == 0 dut.wr_valid.value = 0 - while int(dut.result_valid.value) == 0: - await RisingEdge(dut.clk) - dut.result_ready.value = 1 - await RisingEdge(dut.clk) + # Consume full serial stream so the DUT returns idle + beats = await collect_topk(dut, timeout_cycles=2000) + assert len(beats) > 0 + assert beats[-1][3] == 1 # last asserted # ── Test I: Read noise model match ────────────────────────────────────────── diff --git a/hw/syn/Makefile b/hw/syn/Makefile index 3122f4e..61c620d 100644 --- a/hw/syn/Makefile +++ b/hw/syn/Makefile @@ -16,6 +16,8 @@ TOP ?= cam_top NUM_ROWS ?= 4096 HASH_BITS ?= 512 LANES ?= 8 +TOPK_K ?= 4 +FIFO_DEPTH ?= 16 OUT_DIR ?= build # ── TOP enforcement (only cam_top is supported) ─────────────────────────────── @@ -41,7 +43,7 @@ ARTIFACTS := $(HIER_JSON) $(FLAT_JSON) $(SYNTH_V) $(HIER_LOG) $(FLAT_LOG) # ── Yosys command-line defines (always passed) ──────────────────────────────── # cam_params.svh uses `ifndef guards, so -D on the Yosys CLI overrides defaults. # Use -D NAME=value syntax (Yosys 0.62); do NOT use -define. -YOSYS_DEFINES := -D NUM_ROWS=$(NUM_ROWS) -D HASH_BITS=$(HASH_BITS) -D LANES=$(LANES) +YOSYS_DEFINES := -D NUM_ROWS=$(NUM_ROWS) -D HASH_BITS=$(HASH_BITS) -D LANES=$(LANES) -D TOPK_K=$(TOPK_K) -D FIFO_DEPTH=$(FIFO_DEPTH) # ── Targets ─────────────────────────────────────────────────────────────────── .PHONY: synth hier flat clean diff --git a/hw/syn/synth_cam_top_flat.ys b/hw/syn/synth_cam_top_flat.ys index 5cf1136..ca76468 100644 --- a/hw/syn/synth_cam_top_flat.ys +++ b/hw/syn/synth_cam_top_flat.ys @@ -15,6 +15,9 @@ read_verilog -sv -D SYNTHESIS ../rtl/noise/cam_write_noise.sv read_verilog -sv -D SYNTHESIS ../rtl/noise/cam_read_noise.sv read_verilog -sv -D SYNTHESIS ../rtl/core/cam_core_banked.sv read_verilog -sv -D SYNTHESIS ../rtl/core/popcount_pipeline.sv +read_verilog -sv -D SYNTHESIS ../rtl/core/candidate_fifo.sv +read_verilog -sv -D SYNTHESIS ../rtl/core/topk_tracker.sv +read_verilog -sv -D SYNTHESIS ../rtl/core/result_serializer.sv read_verilog -sv -D SYNTHESIS ../rtl/core/match_engine_pipeline.sv read_verilog -sv -D SYNTHESIS ../rtl/cam_noisy.sv read_verilog -sv -D SYNTHESIS ../rtl/cam_top.sv diff --git a/hw/syn/synth_cam_top_hier.ys b/hw/syn/synth_cam_top_hier.ys index bac2ccc..5dba7ca 100644 --- a/hw/syn/synth_cam_top_hier.ys +++ b/hw/syn/synth_cam_top_hier.ys @@ -15,6 +15,9 @@ read_verilog -sv -D SYNTHESIS ../rtl/noise/cam_write_noise.sv read_verilog -sv -D SYNTHESIS ../rtl/noise/cam_read_noise.sv read_verilog -sv -D SYNTHESIS ../rtl/core/cam_core_banked.sv read_verilog -sv -D SYNTHESIS ../rtl/core/popcount_pipeline.sv +read_verilog -sv -D SYNTHESIS ../rtl/core/candidate_fifo.sv +read_verilog -sv -D SYNTHESIS ../rtl/core/topk_tracker.sv +read_verilog -sv -D SYNTHESIS ../rtl/core/result_serializer.sv read_verilog -sv -D SYNTHESIS ../rtl/core/match_engine_pipeline.sv read_verilog -sv -D SYNTHESIS ../rtl/cam_noisy.sv read_verilog -sv -D SYNTHESIS ../rtl/cam_top.sv