From 28e0c1a49f7c9ea91d72081449e8970e852d9619 Mon Sep 17 00:00:00 2001 From: afiqzudinhadi Date: Thu, 2 Jul 2026 15:39:52 +0800 Subject: [PATCH] =?UTF-8?q?Never=20trim=20dictionary=20=E2=80=94=20hard=20?= =?UTF-8?q?error=20if=20over=20budget?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Removed all trimming/auto-trim logic. Build fails if compiled dict exceeds flash budget. CHD now uses spare slots (1.23x load factor) and fewer buckets (entry_count/5) for reliable construction at any entry count. Binary format bumped to v3: header stores slot_count and entry_count separately. --- Kconfig | 2 +- src/dict_mphf.c | 14 ++--- src/dict_mphf.h | 20 ++++--- tools/compile_mphf.py | 126 +++++++++++++----------------------------- 4 files changed, 59 insertions(+), 103 deletions(-) diff --git a/Kconfig b/Kconfig index b70ba36..5dd3b87 100644 --- a/Kconfig +++ b/Kconfig @@ -78,7 +78,7 @@ config STENO_DICT_MAX_SIZE default 473088 help Max compiled dict size. 473088 = 462KB. - The MPHF compiler auto-trims to fit. + Build fails if dict exceeds this — no trimming. menuconfig STENO_SPLIT_DICT bool "Split dictionary across both halves" diff --git a/src/dict_mphf.c b/src/dict_mphf.c index 8151917..9282ee7 100644 --- a/src/dict_mphf.c +++ b/src/dict_mphf.c @@ -158,15 +158,15 @@ int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len) dict->disp_section_len = align4((disp_bits_total + 7) / 8); offset += dict->disp_section_len; - /* Values */ + /* Values (slot_count slots, not entry_count) */ dict->values = base + offset; - uint32_t val_bits_total = (uint32_t)hdr->entry_count * hdr->value_bits; + uint32_t val_bits_total = (uint32_t)hdr->slot_count * hdr->value_bits; dict->val_section_len = align4((val_bits_total + 7) / 8); offset += dict->val_section_len; - /* Fingerprints */ + /* Fingerprints (one per slot) */ dict->fingerprints = base + offset; - dict->fp_section_len = align4(hdr->entry_count); + dict->fp_section_len = align4(hdr->slot_count); offset += dict->fp_section_len; /* String offsets (u24 LE, 3 bytes each) */ @@ -189,7 +189,7 @@ int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len) } /* Block size from header (0 = legacy default 4096) */ - dict->blk_size = hdr->block_size ? hdr->block_size : DICT_MPHF_BLOCK_SIZE; + dict->blk_size = hdr->block_size ? hdr->block_size : 4096; /* Prefix table at end */ uint32_t prefix_bytes = (uint32_t)hdr->prefix_count * 4; @@ -236,7 +236,7 @@ static const char *resolve_string(const struct dict_mphf *dict, uint32_t val_id) const uint8_t *compressed = dict->blocks_start + blk_start; uint32_t compressed_len = blk_end - blk_start; - static uint8_t decomp_buf[DICT_MPHF_BLOCK_SIZE]; + static uint8_t decomp_buf[4096]; static uint32_t cached_block = UINT32_MAX; static size_t cached_len; @@ -285,7 +285,7 @@ const char *dict_mphf_lookup(const struct dict_mphf *dict, uint32_t d = read_bits(dict->displacements, bucket * (uint32_t)hdr->disp_bits, hdr->disp_bits); - uint32_t slot = hash_key(key_buf, key_len, d + 1) % hdr->entry_count; + uint32_t slot = hash_key(key_buf, key_len, d + 1) % hdr->slot_count; uint8_t expected_fp = (uint8_t)(fnv1a_32(key_buf, key_len) & 0xFF); if (dict->fingerprints[slot] != expected_fp) { diff --git a/src/dict_mphf.h b/src/dict_mphf.h index f5f859e..3bcce8e 100644 --- a/src/dict_mphf.h +++ b/src/dict_mphf.h @@ -1,8 +1,9 @@ /** * MPHF (Minimal Perfect Hash Function) dictionary lookup engine. * - * Binary format v2: CHD MPHF + bit-packed displacements/values + - * fingerprinted verification + block-compressed string table. + * Binary format v3: CHD MPHF with spare slots + bit-packed + * displacements/values + fingerprinted verification + + * block-compressed string table. * * SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 */ @@ -15,24 +16,22 @@ #include #define DICT_MPHF_MAGIC 0x4F4E5453 /* "STNO" */ -#define DICT_MPHF_VERSION 2 +#define DICT_MPHF_VERSION 3 #define DICT_MPHF_FLAG_COMPRESSED 0x0001 -#define DICT_MPHF_BLOCK_SIZE 4096 - struct dict_mphf_header { uint32_t magic; uint16_t version; uint16_t flags; - uint32_t entry_count; + uint32_t slot_count; uint32_t bucket_count; uint32_t unique_count; uint8_t value_bits; uint8_t disp_bits; uint16_t prefix_count; - uint32_t block_size; /* zlib block size (0 = default 4096) */ - uint32_t reserved1; + uint32_t block_size; + uint32_t entry_count; } __attribute__((packed)); _Static_assert(sizeof(struct dict_mphf_header) == 32, "header must be 32 bytes"); @@ -67,4 +66,9 @@ static inline uint32_t dict_mphf_count(const struct dict_mphf *dict) return dict->header->entry_count; } +static inline uint32_t dict_mphf_slot_count(const struct dict_mphf *dict) +{ + return dict->header->slot_count; +} + #endif /* DICT_MPHF_H */ diff --git a/tools/compile_mphf.py b/tools/compile_mphf.py index 8508ea3..b8da0bc 100755 --- a/tools/compile_mphf.py +++ b/tools/compile_mphf.py @@ -166,11 +166,13 @@ def build_chd(keys_and_bytes, entry_count): keys_and_bytes: list of (index, key_bytes) for each entry entry_count: total number of entries - Returns: (displacements, slot_to_entry_idx, max_displacement) + Returns: (displacements, slot_to_entry_idx, slot_count, max_displacement) displacements[bucket] = d value slot_to_entry_idx[slot] = index into keys_and_bytes, or -1 if empty + slot_count: actual number of slots (>= entry_count) """ - bucket_count = max(entry_count, 16) + bucket_count = max(entry_count // 5, 16) + slot_count = math.ceil(entry_count * 1.23) # Assign keys to buckets buckets = defaultdict(list) @@ -183,7 +185,7 @@ def build_chd(keys_and_bytes, entry_count): displacements = [0] * bucket_count occupied = set() - slot_to_entry = [-1] * entry_count + slot_to_entry = [-1] * slot_count max_disp = 0 for bucket_id, members in sorted_buckets: @@ -199,7 +201,7 @@ def build_chd(keys_and_bytes, entry_count): seen = set() for _, kb in member_key_bytes: - slot = hash_key(kb, d + 1) % entry_count + slot = hash_key(kb, d + 1) % slot_count if slot in occupied or slot in seen: collision = True break @@ -223,9 +225,9 @@ def build_chd(keys_and_bytes, entry_count): if not placed: print(f"FATAL: bucket {bucket_id} with {len(members)} keys failed after 65536 tries", file=sys.stderr) - return None, None, None + return None, None, None, None - return displacements, slot_to_entry, max_disp + return displacements, slot_to_entry, slot_count, max_disp # ─── Compilation ─── @@ -258,83 +260,37 @@ def compile_mphf(entries, max_size=None, block_size=4096): keys_and_bytes = list(seen_keys.values()) - # Iteratively trim if needed - while True: - entry_count = len(keys_and_bytes) - if entry_count == 0: - return None + entry_count = len(keys_and_bytes) + if entry_count == 0: + return None - # Build deduped string table - translations = [kb[1] for kb in keys_and_bytes] - unique_translations = sorted(set(translations)) - trans_to_id = {t: i for i, t in enumerate(unique_translations)} - unique_count = len(unique_translations) - - # Estimate size - bucket_count = max(entry_count, 16) - est_value_bits = max(1, math.ceil(math.log2(max(unique_count, 2)))) - est_disp_bits = 16 # conservative - est_disp_bytes = (bucket_count * est_disp_bits + 7) // 8 - est_disp_bytes = ((est_disp_bytes + 3) // 4) * 4 - est_value_bytes = (entry_count * est_value_bits + 7) // 8 - est_value_bytes = ((est_value_bytes + 3) // 4) * 4 - est_fp_bytes = ((entry_count + 3) // 4) * 4 - - # String table (block-compressed) - string_data_raw = b'' - string_offsets = [] - for t in unique_translations: - string_offsets.append(len(string_data_raw)) - string_data_raw += t.encode('utf-8') + b'\x00' - - compressed_blocks = [] - for i in range(0, len(string_data_raw), block_size): - block = string_data_raw[i:i + block_size] - compressed_blocks.append(zlib.compress(block, 9)) - - est_str_offsets = unique_count * 3 # u24 packed LE - est_block_dir = 2 + len(compressed_blocks) * 4 # u16 count + u32 offsets - est_str_data = sum(len(b) for b in compressed_blocks) + est_block_dir - - # Prefix table - prefix_strokes = set() - for kb, trans, stroke_str, strokes in keys_and_bytes: - if len(strokes) > 1: - prefix_strokes.add(strokes[0]) - prefix_list = sorted(prefix_strokes) - est_prefix = len(prefix_list) * 4 - - total_est = 32 + est_disp_bytes + est_value_bytes + est_fp_bytes + est_str_offsets + est_str_data + est_prefix - - if total_est <= max_size: - break - - # Trim: use ratio of overshoot to estimate how many entries to cut - overshoot_ratio = total_est / max_size - target_entries = int(entry_count / overshoot_ratio * 0.98) # 2% safety margin - trim_count = max(1, entry_count - target_entries) - keys_and_bytes = keys_and_bytes[:entry_count - trim_count] - print(f" Trimming to {len(keys_and_bytes)} entries (est {total_est} > {max_size})", - file=sys.stderr) + # Build deduped string table + translations = [kb[1] for kb in keys_and_bytes] + unique_translations = sorted(set(translations)) + trans_to_id = {t: i for i, t in enumerate(unique_translations)} + unique_count = len(unique_translations) entry_count = len(keys_and_bytes) bucket_count = max(entry_count, 16) - print(f" Building CHD MPHF: {entry_count} entries, {bucket_count} buckets...", + print(f" Building CHD MPHF: {entry_count} entries...", file=sys.stderr) # Build CHD chd_input = [(i, keys_and_bytes[i][0]) for i in range(entry_count)] - displacements, slot_to_entry, max_disp = build_chd(chd_input, entry_count) + displacements, slot_to_entry, slot_count, max_disp = build_chd(chd_input, entry_count) if displacements is None: return None + bucket_count = len(displacements) + # Compute actual bit widths disp_bits = max(1, math.ceil(math.log2(max(max_disp + 1, 2)))) value_bits = max(1, math.ceil(math.log2(max(unique_count, 2)))) prefix_count = len(prefix_list) + print(f" Slots: {slot_count}, buckets: {bucket_count}", file=sys.stderr) print(f" Max displacement: {max_disp}, disp_bits: {disp_bits}", file=sys.stderr) print(f" Unique translations: {unique_count}, value_bits: {value_bits}", file=sys.stderr) print(f" Prefix entries: {prefix_count}", file=sys.stderr) @@ -348,10 +304,10 @@ def compile_mphf(entries, max_size=None, block_size=4096): disp_writer.pad_to_alignment(4) disp_section = disp_writer.to_bytes() - # Values section: slot → value_id + # Values section: slot → value_id (slot_count slots, not entry_count) val_writer = BitWriter() - fingerprints = bytearray(entry_count) - for slot in range(entry_count): + fingerprints = bytearray(slot_count) + for slot in range(slot_count): entry_idx = slot_to_entry[slot] if entry_idx >= 0: kb, trans, stroke_str, strokes = keys_and_bytes[entry_idx] @@ -386,21 +342,21 @@ def compile_mphf(entries, max_size=None, block_size=4096): # Header (32 bytes): # magic: u32, version: u16, flags: u16, - # entry_count: u32, bucket_count: u32, unique_count: u32, + # slot_count: u32, bucket_count: u32, unique_count: u32, # value_bits: u8, disp_bits: u8, prefix_count: u16, - # block_size: u32, reserved1: u32 + # block_size: u32, entry_count: u32 header = struct.pack(' max_size: - print(f" WARNING: output {len(binary)} bytes exceeds max {max_size}", file=sys.stderr) + # Check final size — hard error, never trim + if max_size and len(binary) > max_size: + print(f"FATAL: output {len(binary)} bytes exceeds budget {max_size}. " + f"Increase budget or split differently.", file=sys.stderr) + return None return binary, { 'entry_count': entry_count, + 'slot_count': slot_count, 'bucket_count': bucket_count, 'unique_count': unique_count, 'value_bits': value_bits, @@ -489,12 +448,12 @@ def compile_mphf(entries, max_size=None, block_size=4096): def print_stats(stats): """Print size breakdown statistics.""" - print(f"Entries: {stats['entry_count']}") + print(f"Entries: {stats['entry_count']}, slots: {stats.get('slot_count', stats['entry_count'])}") print(f"Block size: {stats.get('block_size', 4096)} bytes") print(f"MPHF displacements: {stats['disp_section_bytes']/1024:.1f} KB " f"({stats['bucket_count']} buckets, {stats['disp_bits']} bits each)") print(f"Value array: {stats['val_section_bytes']/1024:.1f} KB " - f"({stats['entry_count']} entries, {stats['value_bits']} bits each)") + f"({stats.get('slot_count', stats['entry_count'])} slots, {stats['value_bits']} bits each)") print(f"Fingerprints: {stats['fp_section_bytes']/1024:.1f} KB") print(f"String offsets: {stats['str_offsets_bytes']/1024:.1f} KB " f"({stats['unique_count']} unique x 3 bytes)") @@ -534,8 +493,6 @@ def main(): parser.add_argument('output', help='Output binary file') parser.add_argument('--max-size', type=int, default=462*1024, help='Maximum output size in bytes (default: 473088 = 462KB)') - parser.add_argument('--max-entries', type=int, default=None, - help='Maximum number of entries') parser.add_argument('--stats', action='store_true', help='Print size breakdown statistics') parser.add_argument('--verify', action='store_true', default=True, @@ -558,11 +515,6 @@ def main(): entries = list(raw_dict.items()) - if args.max_entries is not None: - entries_scored = sorted(entries, key=lambda e: score_entry(e[0], e[1])) - entries = entries_scored[:args.max_entries] - print(f"Trimmed to {len(entries)} entries (--max-entries)", file=sys.stderr) - # Split-partition mode if args.split_part: left_entries, right_entries = partition_entries(