mirror of
https://github.com/SikongJueluo/Mini-Nav.git
synced 2026-07-12 20:15:31 +08:00
feat(hw/rtl): implement full Top-K CAM search pipeline with serial result output
- add TOPK_K, FIFO_DEPTH, RESULT_SERIAL parameters to cam_params - add candidate_fifo: synchronous ready/valid FIFO for (row, score) candidates - add topk_tracker: tracks top-K candidates with clear/ready handshake - add result_serializer: serializes packed Top-K array into rank-ordered stream - refactor match_engine_pipeline from 4-state to 8-state Top-K pipeline - extend cam_top with serial Top-K interface (result_rank/row/score/last) - add backward-compatible top1_index/top1_score aliases from rank-0 beat - add comprehensive tests for all new modules
This commit is contained in:
124
hw/rtl/core/candidate_fifo.sv
Normal file
124
hw/rtl/core/candidate_fifo.sv
Normal file
@@ -0,0 +1,124 @@
|
||||
`timescale 1ns / 1ps
|
||||
`include "cam_params.svh"
|
||||
|
||||
//==============================================================================
|
||||
// candidate_fifo — Synchronous ready/valid FIFO for (row, score) candidates
|
||||
//
|
||||
// Implements a simple circular-buffer FIFO that accepts writes only when
|
||||
// wr_valid_i && wr_ready_o and reads only when rd_valid_o && rd_ready_i.
|
||||
// Provides empty/full flags. Default depth is `FIFO_DEPTH (16).
|
||||
//
|
||||
// Quality:
|
||||
// - Pointers always wrap at DEPTH-1 (safe for non-power-of-two depths).
|
||||
// - wr_ready_o goes high when full if a read is accepted in the same cycle,
|
||||
// allowing a simultaneous write (throughput-friendly backpressure).
|
||||
// - PTR_BITS is at least 1 for DEPTH=1.
|
||||
//==============================================================================
|
||||
|
||||
module candidate_fifo #(
|
||||
parameter int DEPTH = `FIFO_DEPTH,
|
||||
parameter int ROW_BITS = `ROW_BITS,
|
||||
parameter int SCORE_BITS = `SCORE_BITS,
|
||||
// Ensure at least 1 bit even when DEPTH=1.
|
||||
parameter int PTR_BITS = (DEPTH <= 1) ? 1 : $clog2(DEPTH)
|
||||
) (
|
||||
input logic clk,
|
||||
input logic rst_n,
|
||||
|
||||
// Write interface (producer side)
|
||||
input logic wr_valid_i,
|
||||
output logic wr_ready_o,
|
||||
input logic [ROW_BITS -1:0] wr_row_i,
|
||||
input logic [SCORE_BITS -1:0] wr_score_i,
|
||||
|
||||
// Read interface (consumer side)
|
||||
output logic rd_valid_o,
|
||||
input logic rd_ready_i,
|
||||
output logic [ROW_BITS -1:0] rd_row_o,
|
||||
output logic [SCORE_BITS -1:0] rd_score_o,
|
||||
|
||||
// Status flags
|
||||
output logic empty_o,
|
||||
output logic full_o
|
||||
);
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
// Storage — dual‑port array.
|
||||
//--------------------------------------------------------------------------
|
||||
logic [ROW_BITS -1:0] mem_row [0:DEPTH-1];
|
||||
logic [SCORE_BITS -1:0] mem_score [0:DEPTH-1];
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
// Pointers and fill counter
|
||||
//--------------------------------------------------------------------------
|
||||
logic [PTR_BITS -1:0] wr_ptr, rd_ptr;
|
||||
logic [PTR_BITS :0] count; // extra bit for full detect
|
||||
|
||||
// Wrap-at value (DEPTH-1), sized to match pointer width.
|
||||
// Part-select DEPTH and use a 1'b1 literal so the RHS width equals PTR_BITS.
|
||||
localparam logic [PTR_BITS-1:0] WRAP_AT = DEPTH[PTR_BITS-1:0] - 1'b1;
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
// Status flag outputs (combinational)
|
||||
//--------------------------------------------------------------------------
|
||||
assign empty_o = (count == 0);
|
||||
assign full_o = (count == DEPTH[PTR_BITS:0]);
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
// Handshake outputs
|
||||
//
|
||||
// wr_ready_o is asserted when not full, OR when a read is accepted in
|
||||
// the same cycle (full+pop → slot freed → write can proceed).
|
||||
//--------------------------------------------------------------------------
|
||||
assign wr_ready_o = ~full_o | (rd_valid_o & rd_ready_i);
|
||||
assign rd_valid_o = ~empty_o;
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
// Read data (combinational — value at current rd_ptr)
|
||||
//--------------------------------------------------------------------------
|
||||
assign rd_row_o = mem_row [rd_ptr];
|
||||
assign rd_score_o = mem_score[rd_ptr];
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
// Push / Pop decode
|
||||
//--------------------------------------------------------------------------
|
||||
logic push, pop;
|
||||
assign push = wr_valid_i & wr_ready_o;
|
||||
assign pop = rd_valid_o & rd_ready_i;
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
// Sequential — pointer and count updates
|
||||
//
|
||||
// Pointers wrap at DEPTH-1, safe for any DEPTH (not just powers of two).
|
||||
// Simultaneous push+pop leaves count unchanged.
|
||||
//--------------------------------------------------------------------------
|
||||
always_ff @(posedge clk or negedge rst_n) begin
|
||||
if (!rst_n) begin
|
||||
wr_ptr <= 0;
|
||||
rd_ptr <= 0;
|
||||
count <= 0;
|
||||
end else begin
|
||||
if (push) begin
|
||||
mem_row [wr_ptr] <= wr_row_i;
|
||||
mem_score[wr_ptr] <= wr_score_i;
|
||||
if (wr_ptr == WRAP_AT)
|
||||
wr_ptr <= 0;
|
||||
else
|
||||
wr_ptr <= wr_ptr + 1;
|
||||
end
|
||||
|
||||
if (pop) begin
|
||||
if (rd_ptr == WRAP_AT)
|
||||
rd_ptr <= 0;
|
||||
else
|
||||
rd_ptr <= rd_ptr + 1;
|
||||
end
|
||||
|
||||
// Update fill count
|
||||
if (push & ~pop) count <= count + 1;
|
||||
else if (~push & pop) count <= count - 1;
|
||||
// else: no change (neither, or both)
|
||||
end
|
||||
end
|
||||
|
||||
endmodule
|
||||
@@ -7,10 +7,15 @@ module match_engine_pipeline (
|
||||
input logic query_valid,
|
||||
output logic query_ready,
|
||||
input logic [(`HASH_BITS)-1:0] query_hash,
|
||||
|
||||
// Serial Top-K result interface
|
||||
output logic result_valid,
|
||||
input logic result_ready,
|
||||
output logic [((`TOPK_K <= 1) ? 1 : $clog2(`TOPK_K))-1:0] result_rank,
|
||||
output logic [(`ROW_BITS)-1:0] result_row,
|
||||
output logic [(`SCORE_BITS)-1:0] result_score,
|
||||
output logic result_last,
|
||||
|
||||
output logic busy,
|
||||
output logic rd_valid_o,
|
||||
output logic [(`ROW_BITS)-1:0] rd_base_row_o,
|
||||
@@ -20,35 +25,150 @@ module match_engine_pipeline (
|
||||
input logic [(`LANES)-1:0] rd_lane_valid_i
|
||||
`ifdef SIM_DEBUG
|
||||
,output logic [(`NUM_ROWS)*(`SCORE_BITS)-1:0] score_debug_flat
|
||||
,output logic [($clog2(`NUM_ROWS+1))-1:0] consumed_candidates_debug
|
||||
`endif
|
||||
);
|
||||
typedef enum logic [1:0] {S_IDLE, S_SCAN, S_DRAIN, S_DONE} state_t;
|
||||
//==========================================================================
|
||||
// State encoding
|
||||
//==========================================================================
|
||||
typedef enum logic [2:0] {
|
||||
S_IDLE,
|
||||
S_ISSUE_READ,
|
||||
S_WAIT_SCORE,
|
||||
S_STAGE_CANDIDATES,
|
||||
S_PUSH_CANDIDATES,
|
||||
S_DRAIN,
|
||||
S_SERIALIZE_RESULT,
|
||||
S_DONE
|
||||
} state_t;
|
||||
|
||||
state_t state_q;
|
||||
|
||||
//==========================================================================
|
||||
// Query / read tracking
|
||||
//==========================================================================
|
||||
logic [(`HASH_BITS)-1:0] query_q;
|
||||
logic [(`ROW_BITS)-1:0] issue_base_q;
|
||||
logic [$clog2(`NUM_ROWS/`LANES+1)-1:0] returned_q;
|
||||
logic [(`ROW_BITS)-1:0] best_row_q;
|
||||
logic [(`SCORE_BITS)-1:0] best_score_q;
|
||||
logic [(`ROW_BITS)-1:0] issue_base_q;
|
||||
logic [$clog2(`NUM_ROWS/`LANES+1)-1:0] issued_batches_q;
|
||||
logic [$clog2(`NUM_ROWS/`LANES+1)-1:0] returned_batches_q;
|
||||
logic [$clog2(`NUM_ROWS+1)-1:0] pushed_candidates_q;
|
||||
logic [$clog2(`NUM_ROWS+1)-1:0] consumed_candidates_q;
|
||||
// Total expected valid candidates across all batches
|
||||
logic [$clog2(`NUM_ROWS+1)-1:0] expected_candidates_q;
|
||||
// Batch-level valid-lane mask for return tracking
|
||||
logic [(`LANES)-1:0] batch_lane_mask_q;
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
// Read/noise staging registers (feed popcount pipeline)
|
||||
//--------------------------------------------------------------------------
|
||||
logic rd_stage_valid_q;
|
||||
logic [(`LANES)*(`ROW_BITS)-1:0] rd_stage_row_ids_q;
|
||||
logic [(`LANES)*(`HASH_BITS)-1:0] rd_stage_hashes_q;
|
||||
logic [(`LANES)-1:0] rd_stage_lane_valid_q;
|
||||
|
||||
//--------------------------------------------------------------------------
|
||||
// Candidate staging registers (scored output from popcount, before FIFO)
|
||||
//--------------------------------------------------------------------------
|
||||
logic [(`LANES)-1:0] staged_valid_q;
|
||||
logic [(`LANES)*(`ROW_BITS)-1:0] staged_rows_q;
|
||||
logic [(`LANES)*(`SCORE_BITS)-1:0] staged_scores_q;
|
||||
logic [$clog2(`LANES)-1:0] push_lane_idx_q;
|
||||
|
||||
//==========================================================================
|
||||
// Popcount per-lane wires
|
||||
//==========================================================================
|
||||
logic [(`HASH_BITS)-1:0] match_bits [0:`LANES-1];
|
||||
logic score_valid [0:`LANES-1];
|
||||
logic [(`ROW_BITS)-1:0] score_row [0:`LANES-1];
|
||||
logic [(`SCORE_BITS)-1:0] lane_score [0:`LANES-1];
|
||||
logic [(`ROW_BITS)-1:0] batch_row;
|
||||
logic [(`SCORE_BITS)-1:0] batch_score;
|
||||
|
||||
assign query_ready = (state_q == S_IDLE);
|
||||
assign result_valid = (state_q == S_DONE);
|
||||
assign result_row = best_row_q;
|
||||
assign result_score = best_score_q;
|
||||
assign busy = (state_q != S_IDLE);
|
||||
assign rd_valid_o = (state_q == S_SCAN);
|
||||
assign rd_base_row_o = issue_base_q;
|
||||
//==========================================================================
|
||||
// Candidate FIFO interface
|
||||
//==========================================================================
|
||||
logic cand_valid;
|
||||
logic cand_ready;
|
||||
logic [(`ROW_BITS)-1:0] cand_row;
|
||||
logic [(`SCORE_BITS)-1:0] cand_score;
|
||||
wire cand_fire = cand_valid && cand_ready;
|
||||
|
||||
//==========================================================================
|
||||
// FIFO wires
|
||||
//==========================================================================
|
||||
logic fifo_wr_valid;
|
||||
logic fifo_wr_ready;
|
||||
logic [(`ROW_BITS)-1:0] fifo_wr_row;
|
||||
logic [(`SCORE_BITS)-1:0] fifo_wr_score;
|
||||
logic fifo_rd_valid;
|
||||
logic fifo_rd_ready;
|
||||
logic [(`ROW_BITS)-1:0] fifo_rd_row;
|
||||
logic [(`SCORE_BITS)-1:0] fifo_rd_score;
|
||||
logic fifo_empty;
|
||||
|
||||
//==========================================================================
|
||||
// Tracker wires
|
||||
//==========================================================================
|
||||
logic tracker_clear;
|
||||
logic tracker_candidate_ready;
|
||||
logic [(`TOPK_K)*(`ROW_BITS)-1:0] tracker_topk_rows;
|
||||
logic [(`TOPK_K)*(`SCORE_BITS)-1:0] tracker_topk_scores;
|
||||
logic tracker_update_pending;
|
||||
|
||||
//==========================================================================
|
||||
// Serializer wires
|
||||
//==========================================================================
|
||||
logic serializer_start;
|
||||
logic serializer_busy;
|
||||
logic serializer_done;
|
||||
logic serializer_result_valid;
|
||||
logic [((`TOPK_K <= 1) ? 1 : $clog2(`TOPK_K))-1:0] serializer_result_rank;
|
||||
logic [(`ROW_BITS)-1:0] serializer_result_row;
|
||||
logic [(`SCORE_BITS)-1:0] serializer_result_score;
|
||||
logic serializer_result_last;
|
||||
|
||||
//==========================================================================
|
||||
// Completion condition wires
|
||||
// Width-matched comparison targets to suppress Verilator WIDTHEXPAND.
|
||||
// The assigned constants fit exactly — WIDTHTRUNC is benign.
|
||||
/* verilator lint_off WIDTHTRUNC */
|
||||
wire [$clog2(`NUM_ROWS/`LANES+1)-1:0] num_batches_w = `NUM_ROWS / `LANES;
|
||||
wire [$clog2(`LANES)-1:0] last_lane_w = `LANES - 1;
|
||||
/* verilator lint_on WIDTHTRUNC */
|
||||
|
||||
wire all_reads_issued = (issued_batches_q == num_batches_w);
|
||||
wire all_scores_returned = (returned_batches_q == num_batches_w);
|
||||
wire all_candidates_pushed = (pushed_candidates_q == expected_candidates_q);
|
||||
wire all_candidates_consumed = (consumed_candidates_q == expected_candidates_q);
|
||||
wire scan_drained = all_reads_issued && all_scores_returned
|
||||
&& all_candidates_pushed
|
||||
&& all_candidates_consumed
|
||||
&& fifo_empty
|
||||
&& !tracker_update_pending;
|
||||
|
||||
//==========================================================================
|
||||
// Combinational output assignments
|
||||
//==========================================================================
|
||||
assign query_ready = (state_q == S_IDLE);
|
||||
assign rd_valid_o = (state_q == S_ISSUE_READ);
|
||||
assign rd_base_row_o = issue_base_q;
|
||||
assign busy = (state_q != S_IDLE) || serializer_busy;
|
||||
|
||||
`ifdef SIM_DEBUG
|
||||
assign consumed_candidates_debug = consumed_candidates_q;
|
||||
`endif
|
||||
|
||||
// Result outputs come directly from the serializer
|
||||
assign result_valid = serializer_result_valid;
|
||||
assign result_rank = serializer_result_rank;
|
||||
assign result_row = serializer_result_row;
|
||||
assign result_score = serializer_result_score;
|
||||
assign result_last = serializer_result_last;
|
||||
|
||||
//==========================================================================
|
||||
// Generate: popcount pipeline per lane
|
||||
//==========================================================================
|
||||
generate
|
||||
for (genvar lane = 0; lane < `LANES; lane++) begin : gen_scores
|
||||
assign match_bits[lane] = ~(query_q ^ rd_hashes_i[lane*`HASH_BITS +: `HASH_BITS]);
|
||||
assign match_bits[lane] = ~(query_q ^ rd_stage_hashes_q[lane*`HASH_BITS +: `HASH_BITS]);
|
||||
popcount_pipeline #(
|
||||
.WIDTH(`HASH_BITS),
|
||||
.ROW_BITS(`ROW_BITS),
|
||||
@@ -56,8 +176,8 @@ module match_engine_pipeline (
|
||||
) u_popcount_pipeline (
|
||||
.clk(clk),
|
||||
.rst_n(rst_n),
|
||||
.valid_i(rd_valid_i && rd_lane_valid_i[lane]),
|
||||
.row_i(rd_row_ids_i[lane*`ROW_BITS +: `ROW_BITS]),
|
||||
.valid_i(rd_stage_valid_q && rd_stage_lane_valid_q[lane]),
|
||||
.row_i(rd_stage_row_ids_q[lane*`ROW_BITS +: `ROW_BITS]),
|
||||
.bits_i(match_bits[lane]),
|
||||
.valid_o(score_valid[lane]),
|
||||
.row_o(score_row[lane]),
|
||||
@@ -66,75 +186,284 @@ module match_engine_pipeline (
|
||||
end
|
||||
endgenerate
|
||||
|
||||
always_comb begin
|
||||
batch_row = score_row[0];
|
||||
batch_score = lane_score[0];
|
||||
for (int lane = 1; lane < `LANES; lane++) begin
|
||||
if ((lane_score[lane] > batch_score) ||
|
||||
((lane_score[lane] == batch_score) && (score_row[lane] < batch_row))) begin
|
||||
batch_score = lane_score[lane];
|
||||
batch_row = score_row[lane];
|
||||
//==========================================================================
|
||||
// Batch scores done: true when all valid lanes in the current batch
|
||||
// have their score_valid asserted, or vacuously true for an all-invalid batch.
|
||||
wire batch_scores_done;
|
||||
generate
|
||||
if (`LANES == 1) begin : gen_batch_done_1
|
||||
assign batch_scores_done = rd_stage_lane_valid_q[0]
|
||||
? score_valid[0] : 1'b1;
|
||||
end else begin : gen_batch_done_n
|
||||
logic all_valid_ready;
|
||||
always_comb begin
|
||||
all_valid_ready = 1'b1;
|
||||
for (int l = 0; l < `LANES; l++) begin
|
||||
if (rd_stage_lane_valid_q[l] && !score_valid[l]) begin
|
||||
all_valid_ready = 1'b0;
|
||||
end
|
||||
end
|
||||
end
|
||||
assign batch_scores_done = all_valid_ready;
|
||||
end
|
||||
end
|
||||
endgenerate
|
||||
|
||||
//==========================================================================
|
||||
// Candidate FIFO write interface
|
||||
//==========================================================================
|
||||
assign cand_valid = (state_q == S_PUSH_CANDIDATES)
|
||||
&& staged_valid_q[push_lane_idx_q];
|
||||
assign cand_row = staged_rows_q[push_lane_idx_q * `ROW_BITS +: `ROW_BITS];
|
||||
assign cand_score = staged_scores_q[push_lane_idx_q * `SCORE_BITS +: `SCORE_BITS];
|
||||
assign cand_ready = fifo_wr_ready;
|
||||
|
||||
assign fifo_wr_valid = cand_valid;
|
||||
assign fifo_wr_row = cand_row;
|
||||
assign fifo_wr_score = cand_score;
|
||||
|
||||
//==========================================================================
|
||||
// FIFO output
|
||||
//==========================================================================
|
||||
assign fifo_rd_ready = tracker_candidate_ready;
|
||||
|
||||
wire fifo_rd_fire = fifo_rd_valid && fifo_rd_ready;
|
||||
|
||||
//==========================================================================
|
||||
// Serializer start edge detection
|
||||
//==========================================================================
|
||||
logic start_ser_q1, start_ser_q2;
|
||||
wire start_ser_raw = (state_q == S_DRAIN) && scan_drained;
|
||||
|
||||
always_ff @(posedge clk or negedge rst_n) begin
|
||||
if (!rst_n) begin
|
||||
state_q <= S_IDLE;
|
||||
query_q <= '0;
|
||||
issue_base_q <= '0;
|
||||
returned_q <= 0;
|
||||
best_row_q <= '0;
|
||||
best_score_q <= '0;
|
||||
start_ser_q1 <= 1'b0;
|
||||
start_ser_q2 <= 1'b0;
|
||||
end else begin
|
||||
start_ser_q1 <= start_ser_raw;
|
||||
start_ser_q2 <= start_ser_q1;
|
||||
end
|
||||
end
|
||||
|
||||
assign serializer_start = start_ser_q1 && !start_ser_q2;
|
||||
|
||||
//==========================================================================
|
||||
// Tracker clear: pulse on new query
|
||||
//==========================================================================
|
||||
assign tracker_clear = (state_q == S_IDLE) && query_valid;
|
||||
|
||||
//==========================================================================
|
||||
// Submodule instantiations
|
||||
//==========================================================================
|
||||
|
||||
// --- candidate_fifo (full_o unused) ---
|
||||
/* verilator lint_off PINCONNECTEMPTY */
|
||||
candidate_fifo #(
|
||||
.DEPTH(`FIFO_DEPTH),
|
||||
.ROW_BITS(`ROW_BITS),
|
||||
.SCORE_BITS(`SCORE_BITS)
|
||||
) u_candidate_fifo (
|
||||
.clk(clk),
|
||||
.rst_n(rst_n),
|
||||
.wr_valid_i(fifo_wr_valid),
|
||||
.wr_ready_o(fifo_wr_ready),
|
||||
.wr_row_i(fifo_wr_row),
|
||||
.wr_score_i(fifo_wr_score),
|
||||
.rd_valid_o(fifo_rd_valid),
|
||||
.rd_ready_i(fifo_rd_ready),
|
||||
.rd_row_o(fifo_rd_row),
|
||||
.rd_score_o(fifo_rd_score),
|
||||
.empty_o(fifo_empty),
|
||||
.full_o() // intentionally unused
|
||||
);
|
||||
/* verilator lint_on PINCONNECTEMPTY */
|
||||
|
||||
// --- topk_tracker ---
|
||||
topk_tracker #(
|
||||
.K(`TOPK_K),
|
||||
.ROW_BITS(`ROW_BITS),
|
||||
.SCORE_BITS(`SCORE_BITS)
|
||||
) u_topk_tracker (
|
||||
.clk(clk),
|
||||
.rst_n(rst_n),
|
||||
.clear_i(tracker_clear),
|
||||
.candidate_valid_i(fifo_rd_valid),
|
||||
.candidate_ready_o(tracker_candidate_ready),
|
||||
.candidate_row_i(fifo_rd_row),
|
||||
.candidate_score_i(fifo_rd_score),
|
||||
.topk_rows_o(tracker_topk_rows),
|
||||
.topk_scores_o(tracker_topk_scores),
|
||||
.update_pending_o(tracker_update_pending)
|
||||
);
|
||||
|
||||
// --- result_serializer ---
|
||||
result_serializer #(
|
||||
.K(`TOPK_K),
|
||||
.ROW_BITS(`ROW_BITS),
|
||||
.SCORE_BITS(`SCORE_BITS)
|
||||
) u_result_serializer (
|
||||
.clk(clk),
|
||||
.rst_n(rst_n),
|
||||
.start_i(serializer_start),
|
||||
.busy_o(serializer_busy),
|
||||
.done_o(serializer_done),
|
||||
.topk_rows_i(tracker_topk_rows),
|
||||
.topk_scores_i(tracker_topk_scores),
|
||||
.result_valid_o(serializer_result_valid),
|
||||
.result_ready_i(result_ready),
|
||||
.result_rank_o(serializer_result_rank),
|
||||
.result_row_o(serializer_result_row),
|
||||
.result_score_o(serializer_result_score),
|
||||
.result_last_o(serializer_result_last)
|
||||
);
|
||||
|
||||
//==========================================================================
|
||||
// Sequential: main FSM + registers
|
||||
//==========================================================================
|
||||
always_ff @(posedge clk or negedge rst_n) begin
|
||||
if (!rst_n) begin
|
||||
state_q <= S_IDLE;
|
||||
query_q <= '0;
|
||||
issue_base_q <= '0;
|
||||
issued_batches_q <= 0;
|
||||
returned_batches_q <= 0;
|
||||
pushed_candidates_q <= 0;
|
||||
consumed_candidates_q <= 0;
|
||||
expected_candidates_q <= 0;
|
||||
batch_lane_mask_q <= '0;
|
||||
rd_stage_valid_q <= 1'b0;
|
||||
rd_stage_row_ids_q <= '0;
|
||||
rd_stage_hashes_q <= '0;
|
||||
rd_stage_lane_valid_q <= '0;
|
||||
staged_valid_q <= '0;
|
||||
staged_rows_q <= '0;
|
||||
staged_scores_q <= '0;
|
||||
push_lane_idx_q <= 0;
|
||||
`ifdef SIM_DEBUG
|
||||
score_debug_flat <= '0;
|
||||
score_debug_flat <= '0;
|
||||
`endif
|
||||
end else begin
|
||||
// Defaults for one-cycle flags
|
||||
rd_stage_valid_q <= 1'b0;
|
||||
|
||||
// Consumed candidates counter (FIFO → Tracker handshake)
|
||||
if (fifo_rd_fire) begin
|
||||
consumed_candidates_q <= consumed_candidates_q + 1;
|
||||
end
|
||||
|
||||
// State machine
|
||||
unique case (state_q)
|
||||
|
||||
S_IDLE: begin
|
||||
if (query_valid) begin
|
||||
query_q <= query_hash;
|
||||
issue_base_q <= '0;
|
||||
returned_q <= 0;
|
||||
best_row_q <= TIE_BREAK_SENTINEL;
|
||||
best_score_q <= '0;
|
||||
query_q <= query_hash;
|
||||
issue_base_q <= '0;
|
||||
issued_batches_q <= 0;
|
||||
returned_batches_q <= 0;
|
||||
pushed_candidates_q <= 0;
|
||||
consumed_candidates_q <= 0;
|
||||
expected_candidates_q <= 0;
|
||||
batch_lane_mask_q <= '0;
|
||||
push_lane_idx_q <= 0;
|
||||
staged_valid_q <= '0;
|
||||
`ifdef SIM_DEBUG
|
||||
score_debug_flat <= '0;
|
||||
score_debug_flat <= '0;
|
||||
`endif
|
||||
state_q <= S_SCAN;
|
||||
state_q <= S_ISSUE_READ;
|
||||
end
|
||||
end
|
||||
S_SCAN: begin
|
||||
if (issue_base_q + `LANES >= `NUM_ROWS) begin
|
||||
state_q <= S_DRAIN;
|
||||
end else begin
|
||||
issue_base_q <= issue_base_q + `LANES;
|
||||
|
||||
S_ISSUE_READ: begin
|
||||
if (rd_valid_i) begin
|
||||
// Capture staging data for popcount pipeline
|
||||
rd_stage_valid_q <= 1'b1;
|
||||
rd_stage_row_ids_q <= rd_row_ids_i;
|
||||
rd_stage_hashes_q <= rd_hashes_i;
|
||||
rd_stage_lane_valid_q <= rd_lane_valid_i;
|
||||
// Remember the lane mask for this batch (used in S_WAIT_SCORE
|
||||
// to determine batch_scores_done — survives until overwritten
|
||||
// by the next S_ISSUE_READ).
|
||||
batch_lane_mask_q <= rd_lane_valid_i;
|
||||
issue_base_q <= issue_base_q + `LANES;
|
||||
issued_batches_q <= issued_batches_q + 1;
|
||||
// Accumulate total expected valid candidates
|
||||
expected_candidates_q <= expected_candidates_q
|
||||
+ $countones(rd_lane_valid_i);
|
||||
state_q <= S_WAIT_SCORE;
|
||||
end
|
||||
end
|
||||
|
||||
S_WAIT_SCORE: begin
|
||||
// rd_stage_valid_q already deasserted by default above
|
||||
if (batch_scores_done) begin
|
||||
returned_batches_q <= returned_batches_q + 1;
|
||||
|
||||
// Capture per-lane scores and rows into staging registers
|
||||
for (int lane = 0; lane < `LANES; lane++) begin
|
||||
if (batch_lane_mask_q[lane]) begin
|
||||
staged_valid_q[lane] <= 1'b1;
|
||||
staged_rows_q[lane*`ROW_BITS +: `ROW_BITS] <= score_row[lane];
|
||||
staged_scores_q[lane*`SCORE_BITS +: `SCORE_BITS] <= lane_score[lane];
|
||||
end else begin
|
||||
staged_valid_q[lane] <= 1'b0;
|
||||
end
|
||||
end
|
||||
|
||||
`ifdef SIM_DEBUG
|
||||
for (int lane = 0; lane < `LANES; lane++) begin
|
||||
if (batch_lane_mask_q[lane]) begin
|
||||
score_debug_flat[score_row[lane]*`SCORE_BITS +: `SCORE_BITS]
|
||||
<= lane_score[lane];
|
||||
end
|
||||
end
|
||||
`endif
|
||||
state_q <= S_STAGE_CANDIDATES;
|
||||
end
|
||||
end
|
||||
|
||||
S_STAGE_CANDIDATES: begin
|
||||
state_q <= S_PUSH_CANDIDATES;
|
||||
end
|
||||
|
||||
S_PUSH_CANDIDATES: begin
|
||||
// Advance if the current lane fired or wasn't valid
|
||||
if (cand_fire || !staged_valid_q[push_lane_idx_q]) begin
|
||||
if (cand_fire) begin
|
||||
pushed_candidates_q <= pushed_candidates_q + 1;
|
||||
end
|
||||
|
||||
if (push_lane_idx_q == last_lane_w) begin
|
||||
// Last lane processed — decide next state
|
||||
push_lane_idx_q <= 0;
|
||||
if (all_reads_issued) begin
|
||||
state_q <= S_DRAIN;
|
||||
end else begin
|
||||
state_q <= S_ISSUE_READ;
|
||||
end
|
||||
end else begin
|
||||
push_lane_idx_q <= push_lane_idx_q + 1;
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
S_DRAIN: begin
|
||||
if ((returned_q == (`NUM_ROWS / `LANES)) && !score_valid[0]) begin
|
||||
if (scan_drained) begin
|
||||
state_q <= S_SERIALIZE_RESULT;
|
||||
end
|
||||
end
|
||||
|
||||
S_SERIALIZE_RESULT: begin
|
||||
if (serializer_done) begin
|
||||
state_q <= S_DONE;
|
||||
end
|
||||
end
|
||||
|
||||
S_DONE: begin
|
||||
if (result_ready) state_q <= S_IDLE;
|
||||
state_q <= S_IDLE;
|
||||
end
|
||||
|
||||
default: state_q <= S_IDLE;
|
||||
endcase
|
||||
|
||||
if (score_valid[0]) begin
|
||||
returned_q <= returned_q + 1'b1;
|
||||
if ((batch_score > best_score_q) ||
|
||||
((batch_score == best_score_q) && (batch_row < best_row_q))) begin
|
||||
best_score_q <= batch_score;
|
||||
best_row_q <= batch_row;
|
||||
end
|
||||
`ifdef SIM_DEBUG
|
||||
for (int lane = 0; lane < `LANES; lane++) begin
|
||||
score_debug_flat[score_row[lane]*`SCORE_BITS +: `SCORE_BITS] <= lane_score[lane];
|
||||
end
|
||||
`endif
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
endmodule
|
||||
|
||||
126
hw/rtl/core/result_serializer.sv
Normal file
126
hw/rtl/core/result_serializer.sv
Normal file
@@ -0,0 +1,126 @@
|
||||
`timescale 1ns / 1ps
|
||||
`include "cam_params.svh"
|
||||
|
||||
//==============================================================================
|
||||
// Result Serializer
|
||||
//==============================================================================
|
||||
// Serializes the packed Top-K array (rows and scores) into a rank-ordered
|
||||
// handshake interface, one result per fire.
|
||||
//
|
||||
// Start is edge-triggered (sampled on the cycle start_i is high while not
|
||||
// already active). Once started, the module presents outputs for the
|
||||
// current rank (starting at 0) and advances on each valid-ready fire.
|
||||
// After the last rank (K-1) fires, done_o pulses for one cycle and the
|
||||
// module returns to idle with rank reset to 0.
|
||||
//
|
||||
// Back-pressure: holds current output while result_ready_i is low.
|
||||
//
|
||||
// Supports K=1 cleanly (last rank = rank 0, done on first fire).
|
||||
//==============================================================================
|
||||
|
||||
module result_serializer #(
|
||||
parameter int K = `TOPK_K,
|
||||
parameter int ROW_BITS = `ROW_BITS,
|
||||
parameter int SCORE_BITS = `SCORE_BITS,
|
||||
parameter int RANK_BITS = (K <= 1) ? 1 : $clog2(K)
|
||||
) (
|
||||
input logic clk,
|
||||
input logic rst_n,
|
||||
|
||||
input logic start_i,
|
||||
output logic busy_o,
|
||||
output logic done_o,
|
||||
|
||||
input logic [K*ROW_BITS-1:0] topk_rows_i,
|
||||
input logic [K*SCORE_BITS-1:0] topk_scores_i,
|
||||
|
||||
output logic result_valid_o,
|
||||
input logic result_ready_i,
|
||||
output logic [RANK_BITS-1:0] result_rank_o,
|
||||
output logic [ROW_BITS-1:0] result_row_o,
|
||||
output logic [SCORE_BITS-1:0] result_score_o,
|
||||
output logic result_last_o
|
||||
);
|
||||
|
||||
// ── State encoding ─────────────────────────────────────────────────────
|
||||
typedef enum logic {
|
||||
IDLE,
|
||||
ACTIVE
|
||||
} state_t;
|
||||
|
||||
// ── Localparam for last-rank comparison (width-matched to avoid warnings) ─
|
||||
localparam [RANK_BITS-1:0] LAST_RANK = RANK_BITS'(K - 1);
|
||||
|
||||
state_t state_q, state_next;
|
||||
|
||||
logic [RANK_BITS-1:0] rank_q, rank_next;
|
||||
logic done_q, done_next;
|
||||
|
||||
// Snapshot of packed inputs captured at start (holds during back-pressure)
|
||||
logic [K*ROW_BITS-1:0] rows_snapshot_q, rows_snapshot_next;
|
||||
logic [K*SCORE_BITS-1:0] scores_snapshot_q, scores_snapshot_next;
|
||||
|
||||
// ── Sequential: state, rank, done, and snapshot registers ─────────────
|
||||
always_ff @(posedge clk or negedge rst_n) begin
|
||||
if (!rst_n) begin
|
||||
state_q <= IDLE;
|
||||
rank_q <= '0;
|
||||
done_q <= 1'b0;
|
||||
rows_snapshot_q <= '0;
|
||||
scores_snapshot_q <= '0;
|
||||
end else begin
|
||||
state_q <= state_next;
|
||||
rank_q <= rank_next;
|
||||
done_q <= done_next;
|
||||
rows_snapshot_q <= rows_snapshot_next;
|
||||
scores_snapshot_q <= scores_snapshot_next;
|
||||
end
|
||||
end
|
||||
|
||||
// ── Combinational: next-state, snapshot, and output logic ──────────────
|
||||
always_comb begin
|
||||
// Defaults: stay in current state, retain rank, no done pulse,
|
||||
// hold snapshot (outputs stable under back-pressure)
|
||||
state_next = state_q;
|
||||
rank_next = rank_q;
|
||||
done_next = 1'b0;
|
||||
rows_snapshot_next = rows_snapshot_q;
|
||||
scores_snapshot_next = scores_snapshot_q;
|
||||
|
||||
if (state_q == IDLE) begin
|
||||
// Edge-triggered start — capture packed inputs into snapshot
|
||||
if (start_i) begin
|
||||
state_next = ACTIVE;
|
||||
rank_next = '0; // start from rank 0
|
||||
rows_snapshot_next = topk_rows_i;
|
||||
scores_snapshot_next = topk_scores_i;
|
||||
end
|
||||
end else begin
|
||||
// ACTIVE: advance on fire
|
||||
if (result_valid_o && result_ready_i) begin
|
||||
// Is this the last rank?
|
||||
if (rank_q == LAST_RANK) begin
|
||||
state_next = IDLE;
|
||||
rank_next = '0;
|
||||
done_next = 1'b1;
|
||||
end else begin
|
||||
rank_next = rank_q + 1'b1;
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
// ── Output assignments ─────────────────────────────────────────────────
|
||||
assign busy_o = (state_q == ACTIVE);
|
||||
assign done_o = done_q; // registered — pulses after deactivation
|
||||
assign result_valid_o = (state_q == ACTIVE);
|
||||
|
||||
// Slice the snapshot registers at the current rank
|
||||
assign result_row_o = rows_snapshot_q[rank_q * ROW_BITS +: ROW_BITS];
|
||||
assign result_score_o = scores_snapshot_q[rank_q * SCORE_BITS +: SCORE_BITS];
|
||||
assign result_rank_o = rank_q;
|
||||
|
||||
// result_last_o: high only when active and on the last rank
|
||||
assign result_last_o = (state_q == ACTIVE) && (rank_q == LAST_RANK);
|
||||
|
||||
endmodule
|
||||
126
hw/rtl/core/topk_tracker.sv
Normal file
126
hw/rtl/core/topk_tracker.sv
Normal file
@@ -0,0 +1,126 @@
|
||||
`timescale 1ns / 1ps
|
||||
`include "cam_params.svh"
|
||||
|
||||
//==============================================================================
|
||||
// Top-K Tracker
|
||||
//==============================================================================
|
||||
// Maintains a sorted Top-K array of (row, score) candidates.
|
||||
// Insertion sort on every candidate handshake:
|
||||
// - Higher score wins.
|
||||
// - If equal score, lower row wins (tie-break).
|
||||
// - Entries that are worse than all K current slots are dropped.
|
||||
//
|
||||
// candidate_ready_o is always 1 — the tracker accepts a candidate every cycle.
|
||||
// update_pending_o is always 0 — insertion is fully combinational (one cycle).
|
||||
//
|
||||
// Packed outputs:
|
||||
// topk_rows_o [i*ROW_BITS +: ROW_BITS] = rank i row
|
||||
// topk_scores_o [i*SCORE_BITS +: SCORE_BITS] = rank i score
|
||||
//==============================================================================
|
||||
|
||||
module topk_tracker #(
|
||||
parameter int K = `TOPK_K,
|
||||
parameter int ROW_BITS = `ROW_BITS,
|
||||
parameter int SCORE_BITS = `SCORE_BITS
|
||||
) (
|
||||
input logic clk,
|
||||
input logic rst_n,
|
||||
input logic clear_i,
|
||||
|
||||
input logic candidate_valid_i,
|
||||
output logic candidate_ready_o,
|
||||
input logic [ROW_BITS-1:0] candidate_row_i,
|
||||
input logic [SCORE_BITS-1:0] candidate_score_i,
|
||||
|
||||
output logic [K*ROW_BITS-1:0] topk_rows_o,
|
||||
output logic [K*SCORE_BITS-1:0] topk_scores_o,
|
||||
output logic update_pending_o
|
||||
);
|
||||
|
||||
// ── Sequential state: K-entry sorted arrays ────────────────────────────
|
||||
logic [ROW_BITS-1:0] rows_q [0:K-1];
|
||||
logic [SCORE_BITS-1:0] scores_q [0:K-1];
|
||||
|
||||
// ── Next-state signals ─────────────────────────────────────────────────
|
||||
logic [ROW_BITS-1:0] rows_next [0:K-1];
|
||||
logic [SCORE_BITS-1:0] scores_next [0:K-1];
|
||||
|
||||
// ── Static output assignments ──────────────────────────────────────────
|
||||
assign candidate_ready_o = 1'b1;
|
||||
assign update_pending_o = 1'b0;
|
||||
|
||||
// ── Combinational: insertion logic ─────────────────────────────────────
|
||||
always_comb begin
|
||||
int insert_idx;
|
||||
insert_idx = K; // always assigned — default: no insertion
|
||||
|
||||
// Default: retain current state
|
||||
for (int i = 0; i < K; i++) begin
|
||||
rows_next[i] = rows_q[i];
|
||||
scores_next[i] = scores_q[i];
|
||||
end
|
||||
|
||||
// Only evaluate insertion when a valid candidate is presented
|
||||
if (candidate_valid_i) begin
|
||||
// Find the first slot where candidate is strictly better than the
|
||||
// current occupant. "Better" means higher score, or equal score
|
||||
// with lower row (tie-break).
|
||||
// Note: guard on insert_idx == K avoids loop-local break which
|
||||
// Yosys rejects; the effect is the same — only the first match
|
||||
// is captured because subsequent iterations see insert_idx != K.
|
||||
for (int i = 0; i < K; i++) begin
|
||||
if (insert_idx == K &&
|
||||
(candidate_score_i > scores_q[i] ||
|
||||
(candidate_score_i == scores_q[i] && candidate_row_i < rows_q[i]))) begin
|
||||
insert_idx = i;
|
||||
end
|
||||
end
|
||||
|
||||
// Shift slots down from the insertion point and insert.
|
||||
// Static loop bounds with guard for Yosys compatibility.
|
||||
if (insert_idx < K) begin
|
||||
for (int i = 0; i < K; i++) begin
|
||||
if (i == insert_idx) begin
|
||||
rows_next[i] = candidate_row_i;
|
||||
scores_next[i] = candidate_score_i;
|
||||
end else if (i > insert_idx) begin
|
||||
rows_next[i] = rows_q[i - 1];
|
||||
scores_next[i] = scores_q[i - 1];
|
||||
end
|
||||
// i < insert_idx: keep default from copy loop above
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
// ── Sequential: state update ───────────────────────────────────────────
|
||||
always_ff @(posedge clk or negedge rst_n) begin
|
||||
if (!rst_n) begin
|
||||
for (int i = 0; i < K; i++) begin
|
||||
rows_q[i] <= {ROW_BITS{1'b1}}; // all-ones sentinel
|
||||
scores_q[i] <= '0;
|
||||
end
|
||||
end else if (clear_i) begin
|
||||
for (int i = 0; i < K; i++) begin
|
||||
rows_q[i] <= {ROW_BITS{1'b1}};
|
||||
scores_q[i] <= '0;
|
||||
end
|
||||
end else if (candidate_valid_i) begin
|
||||
for (int i = 0; i < K; i++) begin
|
||||
rows_q[i] <= rows_next[i];
|
||||
scores_q[i] <= scores_next[i];
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
// ── Combinational: pack arrays into flat outputs ───────────────────────
|
||||
always_comb begin
|
||||
topk_rows_o = '0;
|
||||
topk_scores_o = '0;
|
||||
for (int i = 0; i < K; i++) begin
|
||||
topk_rows_o[i*ROW_BITS +: ROW_BITS] = rows_q[i];
|
||||
topk_scores_o[i*SCORE_BITS +: SCORE_BITS] = scores_q[i];
|
||||
end
|
||||
end
|
||||
|
||||
endmodule
|
||||
Reference in New Issue
Block a user