From 1d5bef31299de6d11b62f57aa7f9a8e9f75e41a1 Mon Sep 17 00:00:00 2001 From: afiqzudinhadi Date: Thu, 2 Jul 2026 15:47:23 +0800 Subject: [PATCH] Revert to v2 format, fix CHD with more displacement tries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spare slots (1.23x) doubled binary size → reverted to v2 (no spare slots, entry_count == slot_count). CHD now uses entry_count/4 buckets and 1M displacement tries instead of 65K. No trimming. --- src/dict_mphf.c | 10 +++++----- src/dict_mphf.h | 26 ++++++++++--------------- tools/compile_mphf.py | 45 ++++++++++++++++++++----------------------- 3 files changed, 36 insertions(+), 45 deletions(-) diff --git a/src/dict_mphf.c b/src/dict_mphf.c index 9282ee7..c33cdd2 100644 --- a/src/dict_mphf.c +++ b/src/dict_mphf.c @@ -158,15 +158,15 @@ int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len) dict->disp_section_len = align4((disp_bits_total + 7) / 8); offset += dict->disp_section_len; - /* Values (slot_count slots, not entry_count) */ + /* Values */ dict->values = base + offset; - uint32_t val_bits_total = (uint32_t)hdr->slot_count * hdr->value_bits; + uint32_t val_bits_total = (uint32_t)hdr->entry_count * hdr->value_bits; dict->val_section_len = align4((val_bits_total + 7) / 8); offset += dict->val_section_len; - /* Fingerprints (one per slot) */ + /* Fingerprints */ dict->fingerprints = base + offset; - dict->fp_section_len = align4(hdr->slot_count); + dict->fp_section_len = align4(hdr->entry_count); offset += dict->fp_section_len; /* String offsets (u24 LE, 3 bytes each) */ @@ -285,7 +285,7 @@ const char *dict_mphf_lookup(const struct dict_mphf *dict, uint32_t d = read_bits(dict->displacements, bucket * (uint32_t)hdr->disp_bits, hdr->disp_bits); - uint32_t slot = hash_key(key_buf, key_len, d + 1) % hdr->slot_count; + uint32_t slot = hash_key(key_buf, key_len, d + 1) % hdr->entry_count; uint8_t expected_fp = (uint8_t)(fnv1a_32(key_buf, key_len) & 0xFF); if (dict->fingerprints[slot] != expected_fp) { diff --git a/src/dict_mphf.h b/src/dict_mphf.h index 3bcce8e..7e56393 100644 --- a/src/dict_mphf.h +++ b/src/dict_mphf.h @@ -1,9 +1,8 @@ /** * MPHF (Minimal Perfect Hash Function) dictionary lookup engine. * - * Binary format v3: CHD MPHF with spare slots + bit-packed - * displacements/values + fingerprinted verification + - * block-compressed string table. + * Binary format v2: CHD MPHF + bit-packed displacements/values + + * fingerprinted verification + block-compressed string table. * * SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 */ @@ -16,7 +15,7 @@ #include #define DICT_MPHF_MAGIC 0x4F4E5453 /* "STNO" */ -#define DICT_MPHF_VERSION 3 +#define DICT_MPHF_VERSION 2 #define DICT_MPHF_FLAG_COMPRESSED 0x0001 @@ -24,14 +23,14 @@ struct dict_mphf_header { uint32_t magic; uint16_t version; uint16_t flags; - uint32_t slot_count; + uint32_t entry_count; uint32_t bucket_count; uint32_t unique_count; uint8_t value_bits; uint8_t disp_bits; uint16_t prefix_count; uint32_t block_size; - uint32_t entry_count; + uint32_t reserved1; } __attribute__((packed)); _Static_assert(sizeof(struct dict_mphf_header) == 32, "header must be 32 bytes"); @@ -42,16 +41,16 @@ struct dict_mphf { const uint8_t *values; const uint8_t *fingerprints; const uint8_t *string_offsets; - const uint8_t *str_data_start; /* start of string data section */ + const uint8_t *str_data_start; const uint32_t *prefix_table; uint32_t disp_section_len; uint32_t val_section_len; uint32_t fp_section_len; uint16_t block_count; - const uint32_t *block_dir; /* block offset directory */ - const uint8_t *blocks_start; /* start of compressed blocks */ - uint32_t str_data_len; /* total string data section length */ - uint32_t blk_size; /* actual block size from header */ + const uint32_t *block_dir; + const uint8_t *blocks_start; + uint32_t str_data_len; + uint32_t blk_size; }; int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len); @@ -66,9 +65,4 @@ static inline uint32_t dict_mphf_count(const struct dict_mphf *dict) return dict->header->entry_count; } -static inline uint32_t dict_mphf_slot_count(const struct dict_mphf *dict) -{ - return dict->header->slot_count; -} - #endif /* DICT_MPHF_H */ diff --git a/tools/compile_mphf.py b/tools/compile_mphf.py index 4322d4f..b54576d 100755 --- a/tools/compile_mphf.py +++ b/tools/compile_mphf.py @@ -166,13 +166,11 @@ def build_chd(keys_and_bytes, entry_count): keys_and_bytes: list of (index, key_bytes) for each entry entry_count: total number of entries - Returns: (displacements, slot_to_entry_idx, slot_count, max_displacement) + Returns: (displacements, slot_to_entry_idx, max_displacement) displacements[bucket] = d value slot_to_entry_idx[slot] = index into keys_and_bytes, or -1 if empty - slot_count: actual number of slots (>= entry_count) """ - bucket_count = max(entry_count // 5, 16) - slot_count = math.ceil(entry_count * 1.23) + bucket_count = max(entry_count // 4, 16) # Assign keys to buckets buckets = defaultdict(list) @@ -185,7 +183,7 @@ def build_chd(keys_and_bytes, entry_count): displacements = [0] * bucket_count occupied = set() - slot_to_entry = [-1] * slot_count + slot_to_entry = [-1] * entry_count max_disp = 0 for bucket_id, members in sorted_buckets: @@ -195,13 +193,13 @@ def build_chd(keys_and_bytes, entry_count): member_key_bytes = [(m, keys_and_bytes[m][1]) for m in members] placed = False - for d in range(65536): + for d in range(1 << 20): slots = [] collision = False seen = set() for _, kb in member_key_bytes: - slot = hash_key(kb, d + 1) % slot_count + slot = hash_key(kb, d + 1) % entry_count if slot in occupied or slot in seen: collision = True break @@ -223,11 +221,11 @@ def build_chd(keys_and_bytes, entry_count): break if not placed: - print(f"FATAL: bucket {bucket_id} with {len(members)} keys failed after 65536 tries", + print(f"FATAL: bucket {bucket_id} with {len(members)} keys failed after 1048576 tries", file=sys.stderr) - return None, None, None, None + return None, None, None - return displacements, slot_to_entry, slot_count, max_disp + return displacements, slot_to_entry, max_disp # ─── Compilation ─── @@ -294,7 +292,7 @@ def compile_mphf(entries, max_size=None, block_size=4096): # Build CHD chd_input = [(i, keys_and_bytes[i][0]) for i in range(entry_count)] - displacements, slot_to_entry, slot_count, max_disp = build_chd(chd_input, entry_count) + displacements, slot_to_entry, max_disp = build_chd(chd_input, entry_count) if displacements is None: return None @@ -306,7 +304,7 @@ def compile_mphf(entries, max_size=None, block_size=4096): value_bits = max(1, math.ceil(math.log2(max(unique_count, 2)))) prefix_count = len(prefix_list) - print(f" Slots: {slot_count}, buckets: {bucket_count}", file=sys.stderr) + print(f" Buckets: {bucket_count}", file=sys.stderr) print(f" Max displacement: {max_disp}, disp_bits: {disp_bits}", file=sys.stderr) print(f" Unique translations: {unique_count}, value_bits: {value_bits}", file=sys.stderr) print(f" Prefix entries: {prefix_count}", file=sys.stderr) @@ -320,10 +318,10 @@ def compile_mphf(entries, max_size=None, block_size=4096): disp_writer.pad_to_alignment(4) disp_section = disp_writer.to_bytes() - # Values section: slot → value_id (slot_count slots, not entry_count) + # Values section: slot → value_id val_writer = BitWriter() - fingerprints = bytearray(slot_count) - for slot in range(slot_count): + fingerprints = bytearray(entry_count) + for slot in range(entry_count): entry_idx = slot_to_entry[slot] if entry_idx >= 0: kb, trans, stroke_str, strokes = keys_and_bytes[entry_idx] @@ -358,21 +356,21 @@ def compile_mphf(entries, max_size=None, block_size=4096): # Header (32 bytes): # magic: u32, version: u16, flags: u16, - # slot_count: u32, bucket_count: u32, unique_count: u32, + # entry_count: u32, bucket_count: u32, unique_count: u32, # value_bits: u8, disp_bits: u8, prefix_count: u16, - # block_size: u32, entry_count: u32 + # block_size: u32, reserved1: u32 header = struct.pack('