From ab0e90b84b87fd375afde14fa46dbcd658a96cf7 Mon Sep 17 00:00:00 2001 From: afiqzudinhadi Date: Thu, 2 Jul 2026 15:42:23 +0800 Subject: [PATCH] Fix: restore string table + prefix build removed with trim loop --- tools/compile_mphf.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tools/compile_mphf.py b/tools/compile_mphf.py index b8da0bc..4322d4f 100755 --- a/tools/compile_mphf.py +++ b/tools/compile_mphf.py @@ -270,8 +270,24 @@ def compile_mphf(entries, max_size=None, block_size=4096): trans_to_id = {t: i for i, t in enumerate(unique_translations)} unique_count = len(unique_translations) - entry_count = len(keys_and_bytes) - bucket_count = max(entry_count, 16) + # String table (block-compressed) + string_data_raw = b'' + string_offsets = [] + for t in unique_translations: + string_offsets.append(len(string_data_raw)) + string_data_raw += t.encode('utf-8') + b'\x00' + + compressed_blocks = [] + for i in range(0, len(string_data_raw), block_size): + block = string_data_raw[i:i + block_size] + compressed_blocks.append(zlib.compress(block, 9)) + + # Prefix table + prefix_strokes = set() + for kb, trans, stroke_str, strokes in keys_and_bytes: + if len(strokes) > 1: + prefix_strokes.add(strokes[0]) + prefix_list = sorted(prefix_strokes) print(f" Building CHD MPHF: {entry_count} entries...", file=sys.stderr)