From 1339ec452577b0e3ce7153563a9d985263bd8f46 Mon Sep 17 00:00:00 2001 From: afiqzudinhadi Date: Thu, 2 Jul 2026 15:29:25 +0800 Subject: [PATCH] Fix partition + CHD: empirical bytes/entry, 1:1 bucket ratio MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Partition: ratio-based split overloaded right side (113K entries → 875KB estimate → heavy trimming to 58K). Now uses 4.5 bytes/entry from benchmark data → ~34K left, ~113K right, both fit budgets. CHD: 1:1 bucket ratio (entry_count buckets) prevents hash collision failures on large partitions. Most buckets have 1 key → trivial placement. Displacement array grows but still small. --- tools/compile_mphf.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/tools/compile_mphf.py b/tools/compile_mphf.py index e7d13f2..8508ea3 100755 --- a/tools/compile_mphf.py +++ b/tools/compile_mphf.py @@ -170,7 +170,7 @@ def build_chd(keys_and_bytes, entry_count): displacements[bucket] = d value slot_to_entry_idx[slot] = index into keys_and_bytes, or -1 if empty """ - bucket_count = max(entry_count // 2, min(entry_count, 16)) + bucket_count = max(entry_count, 16) # Assign keys to buckets buckets = defaultdict(list) @@ -271,7 +271,7 @@ def compile_mphf(entries, max_size=None, block_size=4096): unique_count = len(unique_translations) # Estimate size - bucket_count = max(entry_count // 2, min(entry_count, 16)) + bucket_count = max(entry_count, 16) est_value_bits = max(1, math.ceil(math.log2(max(unique_count, 2)))) est_disp_bits = 16 # conservative est_disp_bytes = (bucket_count * est_disp_bits + 7) // 8 @@ -318,7 +318,7 @@ def compile_mphf(entries, max_size=None, block_size=4096): file=sys.stderr) entry_count = len(keys_and_bytes) - bucket_count = max(entry_count // 2, min(entry_count, 16)) + bucket_count = max(entry_count, 16) print(f" Building CHD MPHF: {entry_count} entries, {bucket_count} buckets...", file=sys.stderr) @@ -509,16 +509,17 @@ def partition_entries(entries, left_budget, right_budget): """Partition dict entries by importance into left (central) and right (peripheral). Left gets highest-importance entries first (most common single-stroke words). - Right gets remaining entries. Split point is proportional to flash budgets. - compile_mphf's internal trimming handles any overshoot. + Right gets remaining entries. Uses empirical ~4 bytes/entry from MPHF benchmarks + to estimate how many entries each budget can hold. Returns (left_entries, right_entries). """ sorted_entries = sorted(entries, key=lambda e: score_entry(e[0], e[1])) - total_budget = left_budget + right_budget - left_ratio = left_budget / total_budget - left_max = int(len(sorted_entries) * left_ratio) + # ~4 bytes/entry average from 583KB / 147K entries benchmark + # Use 4.5 for safety margin (smaller partitions have higher per-entry overhead) + est_bytes_per_entry = 4.5 + left_max = int(left_budget / est_bytes_per_entry) left_max = max(1, min(left_max, len(sorted_entries) - 1)) left_entries = sorted_entries[:left_max]