diff --git a/hw/rtl/argmax_update.sv b/hw/rtl/argmax_update.sv new file mode 100644 index 0000000..eaddfcf --- /dev/null +++ b/hw/rtl/argmax_update.sv @@ -0,0 +1,26 @@ +`timescale 1ns/1ps + +module argmax_update #( + parameter int ROW_BITS = 9, + parameter int SCORE_BITS = 10 +) ( + input logic [ROW_BITS-1:0] best_index_i, + input logic [SCORE_BITS-1:0] best_score_i, + + input logic [ROW_BITS-1:0] cand_index_i, + input logic [SCORE_BITS-1:0] cand_score_i, + + output logic [ROW_BITS-1:0] best_index_o, + output logic [SCORE_BITS-1:0] best_score_o +); + always_comb begin + best_index_o = best_index_i; + best_score_o = best_score_i; + + if ((cand_score_i > best_score_i) || + ((cand_score_i == best_score_i) && (cand_index_i < best_index_i))) begin + best_index_o = cand_index_i; + best_score_o = cand_score_i; + end + end +endmodule diff --git a/hw/rtl/cam_core.sv b/hw/rtl/cam_core.sv new file mode 100644 index 0000000..e949ba3 --- /dev/null +++ b/hw/rtl/cam_core.sv @@ -0,0 +1,214 @@ +`timescale 1ns / 1ps + +module cam_core #( + parameter int NUM_ROWS = 512, + parameter int HASH_BITS = 512, + parameter int LANES = 16, + + parameter int ROW_BITS = $clog2(NUM_ROWS), + parameter int SCORE_BITS = $clog2(HASH_BITS + 1) +) ( + input logic clk, + input logic rst_n, + + // Static load interface. In the first prototype, writes are expected + // before online queries begin. + input logic [ ROW_BITS-1:0] wr_row, + input logic [HASH_BITS-1:0] wr_hash, + input logic wr_en, + + // Single-request blocking query interface. + input logic query_valid, + output logic query_ready, + input logic [HASH_BITS-1:0] query_hash, + + output logic result_valid, + input logic result_ready, + output logic [ ROW_BITS-1:0] top1_index, + output logic [SCORE_BITS-1:0] top1_score, + +`ifdef SIM_NOISE + // Flattened for easier cocotb/Verilator handling: + // lane k mask = noise_mask_lanes_flat[k*HASH_BITS +: HASH_BITS] + input logic [LANES*HASH_BITS-1:0] noise_mask_lanes_flat, +`endif + +`ifdef SIM_DEBUG + // Flattened for easier cocotb/Verilator handling: + // score[row] = score_debug_flat[row*SCORE_BITS +: SCORE_BITS] + output logic [NUM_ROWS*SCORE_BITS-1:0] score_debug_flat, +`endif + + output logic busy +); + localparam int NUM_BATCHES = (NUM_ROWS + LANES - 1) / LANES; + + typedef enum logic [1:0] { + S_IDLE, + S_SCAN, + S_DONE + } state_t; + + state_t state_q, state_d; + + logic [HASH_BITS-1:0] cam_mem[NUM_ROWS]; + + logic [HASH_BITS-1: 0] query_q; + logic [ROW_BITS-1 : 0] base_row_q; + logic [ROW_BITS-1 : 0] base_row_d; + + logic [ROW_BITS-1:0] best_index_q, best_index_d; + logic [SCORE_BITS-1:0] best_score_q, best_score_d; + + logic [ ROW_BITS-1:0] lane_best_index; + logic [SCORE_BITS-1:0] lane_best_score; + + logic [ ROW_BITS-1:0] lane_best_index_next[LANES+1]; + logic [SCORE_BITS-1:0] lane_best_score_next[LANES+1]; + + logic [SCORE_BITS-1:0] lane_score [ LANES]; + logic [ ROW_BITS-1:0] lane_row [ LANES]; + logic lane_valid [ LANES]; + + assign query_ready = (state_q == S_IDLE); + assign result_valid = (state_q == S_DONE); + assign top1_index = best_index_q; + assign top1_score = best_score_q; + assign busy = (state_q == S_SCAN); + + // Memory write path. + always_ff @(posedge clk) begin + if (wr_en) begin + cam_mem[wr_row] <= wr_hash; + end + end + +`ifdef SIM_DEBUG + // Clear by default. Individual rows are overwritten during scan. + initial begin + score_debug_flat = '0; + end +`endif + + genvar lane; + generate + for (lane = 0; lane < LANES; lane++) begin : gen_lanes + logic [HASH_BITS-1:0] row_hash; + logic [HASH_BITS-1:0] effective_hash; + logic [HASH_BITS-1:0] match_bits; + + assign lane_row[lane] = base_row_q + lane[ROW_BITS-1:0]; + assign lane_valid[lane] = (lane_row[lane] < NUM_ROWS[ROW_BITS-1:0]); + + // This read is modeled behaviorally. Later this can be replaced + // by banked BRAM/URAM without changing the compare path. + assign row_hash = lane_valid[lane] ? cam_mem[lane_row[lane]] : '0; + +`ifdef SIM_NOISE + logic [HASH_BITS-1:0] lane_noise_mask; + assign lane_noise_mask = noise_mask_lanes_flat[lane*HASH_BITS+:HASH_BITS]; + assign effective_hash = row_hash ^ lane_noise_mask; +`else + assign effective_hash = row_hash; +`endif + + assign match_bits = ~(query_q ^ effective_hash); + + popcount #( + .WIDTH(HASH_BITS), + .GROUP(8), + .OUT_WIDTH(SCORE_BITS) + ) u_popcount ( + .bits_i (match_bits), + .count_o(lane_score[lane]) + ); + + argmax_update #( + .ROW_BITS (ROW_BITS), + .SCORE_BITS(SCORE_BITS) + ) u_argmax_lane ( + .best_index_i(lane_best_index_next[lane]), + .best_score_i(lane_best_score_next[lane]), + .cand_index_i(lane_row[lane]), + .cand_score_i(lane_valid[lane] ? lane_score[lane] : '0), + .best_index_o(lane_best_index_next[lane+1]), + .best_score_o(lane_best_score_next[lane+1]) + ); + end + endgenerate + + assign lane_best_index_next[0] = best_index_q; + assign lane_best_score_next[0] = best_score_q; + assign lane_best_index = lane_best_index_next[LANES]; + assign lane_best_score = lane_best_score_next[LANES]; + + always_comb begin + state_d = state_q; + base_row_d = base_row_q; + best_index_d = best_index_q; + best_score_d = best_score_q; + + unique case (state_q) + S_IDLE: begin + if (query_valid) begin + state_d = S_SCAN; + base_row_d = '0; + best_index_d = {ROW_BITS{1'b1}}; + best_score_d = '0; + end + end + + S_SCAN: begin + best_index_d = lane_best_index; + best_score_d = lane_best_score; + + if (base_row_q + LANES >= NUM_ROWS) begin + state_d = S_DONE; + end else begin + base_row_d = base_row_q + LANES[ROW_BITS-1:0]; + end + end + + S_DONE: begin + if (result_ready) begin + state_d = S_IDLE; + end + end + + default: begin + state_d = S_IDLE; + end + endcase + end + + always_ff @(posedge clk or negedge rst_n) begin + if (!rst_n) begin + state_q <= S_IDLE; + query_q <= '0; + base_row_q <= '0; + best_index_q <= '0; + best_score_q <= '0; + end else begin + state_q <= state_d; + base_row_q <= base_row_d; + best_index_q <= best_index_d; + best_score_q <= best_score_d; + + if ((state_q == S_IDLE) && query_valid) begin + query_q <= query_hash; + end + +`ifdef SIM_DEBUG + if (state_q == S_IDLE && query_valid) begin + score_debug_flat <= '0; + end else if (state_q == S_SCAN) begin + for (int l = 0; l < LANES; l++) begin + if (lane_valid[l]) begin + score_debug_flat[lane_row[l]*SCORE_BITS+:SCORE_BITS] <= lane_score[l]; + end + end + end +`endif + end + end +endmodule diff --git a/hw/rtl/cam_top.sv b/hw/rtl/cam_top.sv new file mode 100644 index 0000000..f6afd82 --- /dev/null +++ b/hw/rtl/cam_top.sv @@ -0,0 +1,64 @@ +`timescale 1ns/1ps + +module cam_top #( + parameter int NUM_ROWS = 512, + parameter int HASH_BITS = 512, + parameter int LANES = 16, + + parameter int ROW_BITS = $clog2(NUM_ROWS), + parameter int SCORE_BITS = $clog2(HASH_BITS + 1) +) ( + input logic clk, + input logic rst_n, + + input logic wr_en, + input logic [ROW_BITS-1:0] wr_row, + input logic [HASH_BITS-1:0] wr_hash, + + input logic query_valid, + output logic query_ready, + input logic [HASH_BITS-1:0] query_hash, + + output logic result_valid, + input logic result_ready, + output logic [ROW_BITS-1:0] top1_index, + output logic [SCORE_BITS-1:0] top1_score, + +`ifdef SIM_NOISE + input logic [LANES*HASH_BITS-1:0] noise_mask_lanes_flat, +`endif + +`ifdef SIM_DEBUG + output logic [NUM_ROWS*SCORE_BITS-1:0] score_debug_flat, +`endif + + output logic busy +); + cam_core #( + .NUM_ROWS(NUM_ROWS), + .HASH_BITS(HASH_BITS), + .LANES(LANES), + .ROW_BITS(ROW_BITS), + .SCORE_BITS(SCORE_BITS) + ) u_core ( + .clk(clk), + .rst_n(rst_n), + .wr_row(wr_row), + .wr_hash(wr_hash), + .wr_en(wr_en), + .query_valid(query_valid), + .query_ready(query_ready), + .query_hash(query_hash), + .result_valid(result_valid), + .result_ready(result_ready), + .top1_index(top1_index), + .top1_score(top1_score), +`ifdef SIM_NOISE + .noise_mask_lanes_flat(noise_mask_lanes_flat), +`endif +`ifdef SIM_DEBUG + .score_debug_flat(score_debug_flat), +`endif + .busy(busy) + ); +endmodule diff --git a/hw/rtl/popcount.sv b/hw/rtl/popcount.sv new file mode 100644 index 0000000..fdc3273 --- /dev/null +++ b/hw/rtl/popcount.sv @@ -0,0 +1,41 @@ +`timescale 1ns/1ps + +module popcount #( + parameter int WIDTH = 512, + parameter int GROUP = 8, + parameter int OUT_WIDTH = $clog2(WIDTH + 1) +) ( + input logic [WIDTH-1:0] bits_i, + output logic [OUT_WIDTH-1:0] count_o +); + localparam int NUM_GROUPS = (WIDTH + GROUP - 1) / GROUP; + localparam int GROUP_COUNT_WIDTH = $clog2(GROUP + 1); + + logic [GROUP_COUNT_WIDTH-1:0] group_counts [NUM_GROUPS]; + + genvar g; + generate + for (g = 0; g < NUM_GROUPS; g++) begin : gen_group_popcount + always_comb begin + group_counts[g] = '0; + for (int b = 0; b < GROUP; b++) begin + int idx; + idx = g * GROUP + b; + if (idx < WIDTH) begin + group_counts[g] = group_counts[g] + bits_i[idx]; + end + end + end + end + endgenerate + + // Reduction over small group counts. This is intentionally simple and + // synthesis-friendly enough for the first prototype. If timing fails, + // replace this block with a fully staged adder tree. + always_comb begin + count_o = '0; + for (int i = 0; i < NUM_GROUPS; i++) begin + count_o = count_o + group_counts[i]; + end + end +endmodule diff --git a/hw/sim/Makefile b/hw/sim/Makefile new file mode 100644 index 0000000..4b7633f --- /dev/null +++ b/hw/sim/Makefile @@ -0,0 +1,30 @@ +# Minimal cocotb Makefile. +# Examples: +# make TESTCASE=basic_write_query_no_noise +# make TESTCASE=external_noise_mask EXTRA_DEFINES="+define+SIM_NOISE +define+SIM_DEBUG" +# +# Verilator is preferred. Icarus may not support all SystemVerilog constructs used here. + +SIM ?= verilator +TOPLEVEL_LANG ?= verilog +TOPLEVEL := cam_top +MODULE ?= tests.test_cam_basic + +NUM_ROWS ?= 512 +HASH_BITS ?= 512 +LANES ?= 16 + +EXTRA_ARGS += -DNUM_ROWS=$(NUM_ROWS) -DHASH_BITS=$(HASH_BITS) -DLANES=$(LANES) + +# cocotb passes PLUSARGS/EXTRA_ARGS differently across simulators. Keep +# SystemVerilog parameters explicit through COMPILE_ARGS for Verilator. +COMPILE_ARGS += -Wall -Wno-fatal +COMPILE_ARGS += +define+SIM_DEBUG +COMPILE_ARGS += $(EXTRA_DEFINES) + +VERILOG_SOURCES += $(PWD)/../rtl/popcount.sv +VERILOG_SOURCES += $(PWD)/../rtl/argmax_update.sv +VERILOG_SOURCES += $(PWD)/../rtl/cam_core.sv +VERILOG_SOURCES += $(PWD)/../rtl/cam_top.sv + +include $(shell cocotb-config --makefiles)/Makefile.sim diff --git a/hw/sim/model/ref_model.py b/hw/sim/model/ref_model.py new file mode 100644 index 0000000..db3ae31 --- /dev/null +++ b/hw/sim/model/ref_model.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Iterable, Sequence +import numpy as np + + +@dataclass(frozen=True) +class MatchResult: + top1_index: int + top1_score: int + scores: np.ndarray + + +def popcount_int(x: int) -> int: + return int(x.bit_count()) + + +def mask_width(width: int) -> int: + return (1 << width) - 1 + + +def xnor_popcount_score(query: int, stored: int, width: int = 512) -> int: + same_bits = ~(query ^ stored) & mask_width(width) + return popcount_int(same_bits) + + +def apply_noise(stored: int, noise_mask: int) -> int: + return stored ^ noise_mask + + +def match_top1( + query: int, + rows: Sequence[int], + *, + width: int = 512, + noise_masks: Sequence[int] | None = None, +) -> MatchResult: + scores = np.zeros(len(rows), dtype=np.int32) + + best_index = 0 + best_score = -1 + + for idx, row in enumerate(rows): + effective = row if noise_masks is None else apply_noise(row, int(noise_masks[idx])) + score = xnor_popcount_score(int(query), int(effective), width) + scores[idx] = score + + # Tie-break: choose the smallest row index. + if score > best_score: + best_score = score + best_index = idx + + return MatchResult( + top1_index=int(best_index), + top1_score=int(best_score), + scores=scores, + ) + + +def random_hashes( + rng: np.random.Generator, + n: int, + *, + width: int = 512, +) -> list[int]: + words = (width + 63) // 64 + out: list[int] = [] + + for _ in range(n): + value = 0 + for w in range(words): + value |= int(rng.integers(0, 1 << 64, dtype=np.uint64)) << (64 * w) + out.append(value & mask_width(width)) + + return out + + +def random_noise_masks( + rng: np.random.Generator, + n: int, + *, + width: int = 512, + bit_flip_rate: float = 0.0, +) -> list[int]: + if not (0.0 <= bit_flip_rate <= 1.0): + raise ValueError("bit_flip_rate must be in [0, 1]") + + masks: list[int] = [] + for _ in range(n): + bits = rng.random(width) < bit_flip_rate + value = 0 + for i, bit in enumerate(bits): + if bool(bit): + value |= 1 << i + masks.append(value) + return masks + + +def pack_lanes_flat(masks: Sequence[int], *, width: int = 512) -> int: + flat = 0 + lane_mask = mask_width(width) + for lane, mask in enumerate(masks): + flat |= (int(mask) & lane_mask) << (lane * width) + return flat + + +def unpack_score_debug_flat(flat: int, num_rows: int, score_bits: int) -> np.ndarray: + mask = (1 << score_bits) - 1 + return np.array( + [(int(flat) >> (row * score_bits)) & mask for row in range(num_rows)], + dtype=np.int32, + ) + + +def split_hash_to_words_le(value: int, *, width: int = 512, word_bits: int = 32) -> list[int]: + n_words = width // word_bits + word_mask = (1 << word_bits) - 1 + return [(int(value) >> (word_bits * i)) & word_mask for i in range(n_words)] + + +def join_hash_words_le(words: Sequence[int], *, word_bits: int = 32) -> int: + value = 0 + word_mask = (1 << word_bits) - 1 + for i, word in enumerate(words): + value |= (int(word) & word_mask) << (word_bits * i) + return value diff --git a/hw/sim/sweep_noise.py b/hw/sim/sweep_noise.py new file mode 100644 index 0000000..ff5e4f6 --- /dev/null +++ b/hw/sim/sweep_noise.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +import argparse + +import numpy as np +from model.ref_model import match_top1, random_hashes, random_noise_masks + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--rows", type=int, default=512) + parser.add_argument("--queries", type=int, default=128) + parser.add_argument("--width", type=int, default=512) + parser.add_argument("--seed", type=int, default=1234) + parser.add_argument( + "--rates", + type=float, + nargs="+", + default=[0.0, 0.001, 0.005, 0.01, 0.02, 0.05], + ) + args = parser.parse_args() + + rng = np.random.default_rng(args.seed) + rows = random_hashes(rng, args.rows, width=args.width) + + # Construct simple positive queries by selecting existing rows. + query_indices = rng.integers(0, args.rows, size=args.queries) + queries = [rows[int(i)] for i in query_indices] + + clean_results = [match_top1(q, rows, width=args.width) for q in queries] + + print("rate,top1_stability,avg_clean_margin") + for rate in args.rates: + stable = 0 + margins = [] + + for q, clean in zip(queries, clean_results): + noise_masks = random_noise_masks( + rng, + args.rows, + width=args.width, + bit_flip_rate=rate, + ) + noisy = match_top1(q, rows, width=args.width, noise_masks=noise_masks) + + if noisy.top1_index == clean.top1_index: + stable += 1 + + sorted_scores = np.sort(clean.scores) + margin = int(sorted_scores[-1] - sorted_scores[-2]) + margins.append(margin) + + print(f"{rate},{stable / args.queries:.6f},{np.mean(margins):.3f}") + + +if __name__ == "__main__": + main() diff --git a/hw/sim/tests/test_cam_basic.py b/hw/sim/tests/test_cam_basic.py new file mode 100644 index 0000000..845d5d0 --- /dev/null +++ b/hw/sim/tests/test_cam_basic.py @@ -0,0 +1,210 @@ +from __future__ import annotations + +import cocotb +import numpy as np +from cocotb.clock import Clock +from cocotb.triggers import RisingEdge +from model.ref_model import ( # noqa: E402 + match_top1, + pack_lanes_flat, + random_hashes, + unpack_score_debug_flat, +) + +NUM_ROWS = 512 +HASH_BITS = 512 +LANES = 16 +SCORE_BITS = 10 + + +async def reset_dut(dut): + dut.rst_n.value = 0 + dut.wr_en.value = 0 + dut.wr_row.value = 0 + dut.wr_hash.value = 0 + dut.query_valid.value = 0 + dut.query_hash.value = 0 + dut.result_ready.value = 1 + + if hasattr(dut, "noise_mask_lanes_flat"): + dut.noise_mask_lanes_flat.value = 0 + + for _ in range(5): + await RisingEdge(dut.clk) + + dut.rst_n.value = 1 + + for _ in range(2): + await RisingEdge(dut.clk) + + +async def write_rows(dut, rows): + for idx, value in enumerate(rows): + dut.wr_row.value = idx + dut.wr_hash.value = int(value) + dut.wr_en.value = 1 + await RisingEdge(dut.clk) + + dut.wr_en.value = 0 + await RisingEdge(dut.clk) + + +async def query_once(dut, query, noise_masks=None): + dut.query_hash.value = int(query) + dut.query_valid.value = 1 + + await RisingEdge(dut.clk) + dut.query_valid.value = 0 + + # Feed lane noise masks batch by batch while DUT is scanning. + # For no-noise builds this signal is absent and ignored. + base = 0 + while int(dut.result_valid.value) == 0: + if hasattr(dut, "noise_mask_lanes_flat") and noise_masks is not None: + lane_masks = [] + for lane in range(LANES): + row = base + lane + lane_masks.append(noise_masks[row] if row < NUM_ROWS else 0) + dut.noise_mask_lanes_flat.value = pack_lanes_flat( + lane_masks, width=HASH_BITS + ) + base += LANES + + await RisingEdge(dut.clk) + + top1_index = int(dut.top1_index.value) + top1_score = int(dut.top1_score.value) + + score_debug = None + if hasattr(dut, "score_debug_flat"): + score_debug = unpack_score_debug_flat( + int(dut.score_debug_flat.value), + NUM_ROWS, + SCORE_BITS, + ) + + await RisingEdge(dut.clk) + return top1_index, top1_score, score_debug + + +@cocotb.test() +async def basic_write_query_no_noise(dut): + cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) + await reset_dut(dut) + + rng = np.random.default_rng(1) + rows = random_hashes(rng, NUM_ROWS, width=HASH_BITS) + query_index = 123 + query = rows[query_index] + + await write_rows(dut, rows) + top1_index, top1_score, score_debug = await query_once(dut, query) + + expected = match_top1(query, rows, width=HASH_BITS) + + assert top1_index == expected.top1_index + assert top1_score == expected.top1_score + assert top1_index == query_index + assert top1_score == HASH_BITS + + if score_debug is not None: + assert np.array_equal(score_debug, expected.scores) + + +@cocotb.test() +async def all_zero_all_one_boundary(dut): + cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) + await reset_dut(dut) + + rows = [0] * NUM_ROWS + rows[0] = 0 + rows[1] = (1 << HASH_BITS) - 1 + + query = 0 + await write_rows(dut, rows) + top1_index, top1_score, score_debug = await query_once(dut, query) + + assert top1_score == HASH_BITS + assert top1_index == 0 + + if score_debug is not None: + assert int(score_debug[0]) == HASH_BITS + assert int(score_debug[1]) == 0 + + +@cocotb.test() +async def known_hamming_distance(dut): + cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) + await reset_dut(dut) + + query = 0 + rows = [0] * NUM_ROWS + rows[10] = (1 << 7) - 1 # Hamming distance = 7 + rows[11] = (1 << 31) - 1 # Hamming distance = 31 + rows[12] = (1 << 128) - 1 # Hamming distance = 128 + + await write_rows(dut, rows) + top1_index, top1_score, score_debug = await query_once(dut, query) + + # Many rows are identical to query; tie-break must select row 0. + assert top1_index == 0 + assert top1_score == HASH_BITS + + if score_debug is not None: + assert int(score_debug[10]) == HASH_BITS - 7 + assert int(score_debug[11]) == HASH_BITS - 31 + assert int(score_debug[12]) == HASH_BITS - 128 + + +@cocotb.test() +async def tie_break_policy(dut): + cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) + await reset_dut(dut) + + rng = np.random.default_rng(2) + rows = random_hashes(rng, NUM_ROWS, width=HASH_BITS) + query = rows[200] + rows[10] = query + rows[20] = query + rows[200] = query + + await write_rows(dut, rows) + top1_index, top1_score, _ = await query_once(dut, query) + + assert top1_index == 10 + assert top1_score == HASH_BITS + + +@cocotb.test() +async def external_noise_mask(dut): + # This test is meaningful only when compiled with SIM_NOISE and SIM_DEBUG. + cocotb.start_soon(Clock(dut.clk, 10, unit="ns").start()) + await reset_dut(dut) + + if not hasattr(dut, "noise_mask_lanes_flat"): + dut._log.warning("SIM_NOISE not enabled; skipping exact noise-mask behavior.") + return + + rng = np.random.default_rng(3) + rows = random_hashes(rng, NUM_ROWS, width=HASH_BITS) + query_index = 42 + query = rows[query_index] + + noise_masks = [0] * NUM_ROWS + noise_masks[query_index] = (1 << 13) - 1 # flip exactly 13 bits + + await write_rows(dut, rows) + top1_index, top1_score, score_debug = await query_once( + dut, + query, + noise_masks=noise_masks, + ) + + expected = match_top1(query, rows, width=HASH_BITS, noise_masks=noise_masks) + + assert top1_index == expected.top1_index + assert top1_score == expected.top1_score + + if score_debug is not None: + assert int(score_debug[query_index]) == HASH_BITS - 13 + assert np.array_equal(score_debug, expected.scores)