afiqzudinhadi/zmk-steno-engine
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Never trim dictionary — hard error if over budget

Browse source 
Removed all trimming/auto-trim logic. Build fails if compiled dict
exceeds flash budget. CHD now uses spare slots (1.23x load factor)
and fewer buckets (entry_count/5) for reliable construction at any
entry count. Binary format bumped to v3: header stores slot_count
and entry_count separately.
This commit is contained in:
afiqzudinhadi 2026-07-02 15:39:52 +08:00
parent 1339ec4525
commit 28e0c1a49f
4 changed files with 59 additions and 103 deletions
Download patch file Download diff file Expand all files Collapse all files

2
Kconfig
View file

@ -78,7 +78,7 @@ config STENO_DICT_MAX_SIZE
default 473088
help
Max compiled dict size. 473088 = 462KB.
The MPHF compiler auto-trims to fit.
Build fails if dict exceeds this — no trimming.
menuconfig STENO_SPLIT_DICT
bool "Split dictionary across both halves"

14
src/dict_mphf.c
View file

@ -158,15 +158,15 @@ int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len)
dict->disp_section_len = align4((disp_bits_total + 7) / 8);
offset += dict->disp_section_len;
/* Values */
/* Values (slot_count slots, not entry_count) */
dict->values = base + offset;
uint32_t val_bits_total = (uint32_t)hdr->entry_count * hdr->value_bits;
uint32_t val_bits_total = (uint32_t)hdr->slot_count * hdr->value_bits;
dict->val_section_len = align4((val_bits_total + 7) / 8);
offset += dict->val_section_len;
/* Fingerprints */
/* Fingerprints (one per slot) */
dict->fingerprints = base + offset;
dict->fp_section_len = align4(hdr->entry_count);
dict->fp_section_len = align4(hdr->slot_count);
offset += dict->fp_section_len;
/* String offsets (u24 LE, 3 bytes each) */
@ -189,7 +189,7 @@ int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len)
}
/* Block size from header (0 = legacy default 4096) */
dict->blk_size = hdr->block_size ? hdr->block_size : DICT_MPHF_BLOCK_SIZE;
dict->blk_size = hdr->block_size ? hdr->block_size : 4096;
/* Prefix table at end */
uint32_t prefix_bytes = (uint32_t)hdr->prefix_count * 4;
@ -236,7 +236,7 @@ static const char *resolve_string(const struct dict_mphf *dict, uint32_t val_id)
const uint8_t *compressed = dict->blocks_start + blk_start;
uint32_t compressed_len = blk_end - blk_start;
static uint8_t decomp_buf[DICT_MPHF_BLOCK_SIZE];
static uint8_t decomp_buf[4096];
static uint32_t cached_block = UINT32_MAX;
static size_t cached_len;
@ -285,7 +285,7 @@ const char *dict_mphf_lookup(const struct dict_mphf *dict,
uint32_t d = read_bits(dict->displacements,
bucket * (uint32_t)hdr->disp_bits,
hdr->disp_bits);
uint32_t slot = hash_key(key_buf, key_len, d + 1) % hdr->entry_count;
uint32_t slot = hash_key(key_buf, key_len, d + 1) % hdr->slot_count;
uint8_t expected_fp = (uint8_t)(fnv1a_32(key_buf, key_len) & 0xFF);
if (dict->fingerprints[slot] != expected_fp) {

20
src/dict_mphf.h
View file

@ -1,8 +1,9 @@
/**
* MPHF (Minimal Perfect Hash Function) dictionary lookup engine.
*
* Binary format v2: CHD MPHF + bit-packed displacements/values +
* fingerprinted verification + block-compressed string table.
* Binary format v3: CHD MPHF with spare slots + bit-packed
* displacements/values + fingerprinted verification +
* block-compressed string table.
*
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*/
@ -15,24 +16,22 @@
#include <stddef.h>
#define DICT_MPHF_MAGIC 0x4F4E5453 /* "STNO" */
#define DICT_MPHF_VERSION 2
#define DICT_MPHF_VERSION 3
#define DICT_MPHF_FLAG_COMPRESSED 0x0001
#define DICT_MPHF_BLOCK_SIZE 4096
struct dict_mphf_header {
uint32_t magic;
uint16_t version;
uint16_t flags;
uint32_t entry_count;
uint32_t slot_count;
uint32_t bucket_count;
uint32_t unique_count;
uint8_t value_bits;
uint8_t disp_bits;
uint16_t prefix_count;
uint32_t block_size; /* zlib block size (0 = default 4096) */
uint32_t reserved1;
uint32_t block_size;
uint32_t entry_count;
} __attribute__((packed));
_Static_assert(sizeof(struct dict_mphf_header) == 32, "header must be 32 bytes");
@ -67,4 +66,9 @@ static inline uint32_t dict_mphf_count(const struct dict_mphf *dict)
return dict->header->entry_count;
}
static inline uint32_t dict_mphf_slot_count(const struct dict_mphf *dict)
{
return dict->header->slot_count;
}
#endif /* DICT_MPHF_H */

126
tools/compile_mphf.py
View file

@ -166,11 +166,13 @@ def build_chd(keys_and_bytes, entry_count):
keys_and_bytes: list of (index, key_bytes) for each entry
entry_count: total number of entries
Returns: (displacements, slot_to_entry_idx, max_displacement)
Returns: (displacements, slot_to_entry_idx, slot_count, max_displacement)
displacements[bucket] = d value
slot_to_entry_idx[slot] = index into keys_and_bytes, or -1 if empty
slot_count: actual number of slots (>= entry_count)
"""
bucket_count = max(entry_count, 16)
bucket_count = max(entry_count // 5, 16)
slot_count = math.ceil(entry_count * 1.23)
# Assign keys to buckets
buckets = defaultdict(list)
@ -183,7 +185,7 @@ def build_chd(keys_and_bytes, entry_count):
displacements = [0] * bucket_count
occupied = set()
slot_to_entry = [-1] * entry_count
slot_to_entry = [-1] * slot_count
max_disp = 0
for bucket_id, members in sorted_buckets:
@ -199,7 +201,7 @@ def build_chd(keys_and_bytes, entry_count):
seen = set()
for _, kb in member_key_bytes:
slot = hash_key(kb, d + 1) % entry_count
slot = hash_key(kb, d + 1) % slot_count
if slot in occupied or slot in seen:
collision = True
break
@ -223,9 +225,9 @@ def build_chd(keys_and_bytes, entry_count):
if not placed:
print(f"FATAL: bucket {bucket_id} with {len(members)} keys failed after 65536 tries",
file=sys.stderr)
return None, None, None
return None, None, None, None
return displacements, slot_to_entry, max_disp
return displacements, slot_to_entry, slot_count, max_disp
# ─── Compilation ───
@ -258,83 +260,37 @@ def compile_mphf(entries, max_size=None, block_size=4096):
keys_and_bytes = list(seen_keys.values())
# Iteratively trim if needed
while True:
entry_count = len(keys_and_bytes)
if entry_count == 0:
return None
entry_count = len(keys_and_bytes)
if entry_count == 0:
return None
# Build deduped string table
translations = [kb[1] for kb in keys_and_bytes]
unique_translations = sorted(set(translations))
trans_to_id = {t: i for i, t in enumerate(unique_translations)}
unique_count = len(unique_translations)
# Estimate size
bucket_count = max(entry_count, 16)
est_value_bits = max(1, math.ceil(math.log2(max(unique_count, 2))))
est_disp_bits = 16 # conservative
est_disp_bytes = (bucket_count * est_disp_bits + 7) // 8
est_disp_bytes = ((est_disp_bytes + 3) // 4) * 4
est_value_bytes = (entry_count * est_value_bits + 7) // 8
est_value_bytes = ((est_value_bytes + 3) // 4) * 4
est_fp_bytes = ((entry_count + 3) // 4) * 4
# String table (block-compressed)
string_data_raw = b''
string_offsets = []
for t in unique_translations:
string_offsets.append(len(string_data_raw))
string_data_raw += t.encode('utf-8') + b'\x00'
compressed_blocks = []
for i in range(0, len(string_data_raw), block_size):
block = string_data_raw[i:i + block_size]
compressed_blocks.append(zlib.compress(block, 9))
est_str_offsets = unique_count * 3 # u24 packed LE
est_block_dir = 2 + len(compressed_blocks) * 4 # u16 count + u32 offsets
est_str_data = sum(len(b) for b in compressed_blocks) + est_block_dir
# Prefix table
prefix_strokes = set()
for kb, trans, stroke_str, strokes in keys_and_bytes:
if len(strokes) > 1:
prefix_strokes.add(strokes[0])
prefix_list = sorted(prefix_strokes)
est_prefix = len(prefix_list) * 4
total_est = 32 + est_disp_bytes + est_value_bytes + est_fp_bytes + est_str_offsets + est_str_data + est_prefix
if total_est <= max_size:
break
# Trim: use ratio of overshoot to estimate how many entries to cut
overshoot_ratio = total_est / max_size
target_entries = int(entry_count / overshoot_ratio * 0.98) # 2% safety margin
trim_count = max(1, entry_count - target_entries)
keys_and_bytes = keys_and_bytes[:entry_count - trim_count]
print(f" Trimming to {len(keys_and_bytes)} entries (est {total_est} > {max_size})",
file=sys.stderr)
# Build deduped string table
translations = [kb[1] for kb in keys_and_bytes]
unique_translations = sorted(set(translations))
trans_to_id = {t: i for i, t in enumerate(unique_translations)}
unique_count = len(unique_translations)
entry_count = len(keys_and_bytes)
bucket_count = max(entry_count, 16)
print(f" Building CHD MPHF: {entry_count} entries, {bucket_count} buckets...",
print(f" Building CHD MPHF: {entry_count} entries...",
file=sys.stderr)
# Build CHD
chd_input = [(i, keys_and_bytes[i][0]) for i in range(entry_count)]
displacements, slot_to_entry, max_disp = build_chd(chd_input, entry_count)
displacements, slot_to_entry, slot_count, max_disp = build_chd(chd_input, entry_count)
if displacements is None:
return None
bucket_count = len(displacements)
# Compute actual bit widths
disp_bits = max(1, math.ceil(math.log2(max(max_disp + 1, 2))))
value_bits = max(1, math.ceil(math.log2(max(unique_count, 2))))
prefix_count = len(prefix_list)
print(f" Slots: {slot_count}, buckets: {bucket_count}", file=sys.stderr)
print(f" Max displacement: {max_disp}, disp_bits: {disp_bits}", file=sys.stderr)
print(f" Unique translations: {unique_count}, value_bits: {value_bits}", file=sys.stderr)
print(f" Prefix entries: {prefix_count}", file=sys.stderr)
@ -348,10 +304,10 @@ def compile_mphf(entries, max_size=None, block_size=4096):
disp_writer.pad_to_alignment(4)
disp_section = disp_writer.to_bytes()
# Values section: slot → value_id
# Values section: slot → value_id (slot_count slots, not entry_count)
val_writer = BitWriter()
fingerprints = bytearray(entry_count)
for slot in range(entry_count):
fingerprints = bytearray(slot_count)
for slot in range(slot_count):
entry_idx = slot_to_entry[slot]
if entry_idx >= 0:
kb, trans, stroke_str, strokes = keys_and_bytes[entry_idx]
@ -386,21 +342,21 @@ def compile_mphf(entries, max_size=None, block_size=4096):
# Header (32 bytes):
# magic: u32, version: u16, flags: u16,
# entry_count: u32, bucket_count: u32, unique_count: u32,
# slot_count: u32, bucket_count: u32, unique_count: u32,
# value_bits: u8, disp_bits: u8, prefix_count: u16,
# block_size: u32, reserved1: u32
# block_size: u32, entry_count: u32
header = struct.pack('<IHHIIIBBHii',
0x4F4E5453, # magic "STNO"
2, # version
3, # version (bumped: slot_count in header)
0x0001, # flags: bit 0 = block-compressed strings
entry_count, # entry_count
slot_count, # slot_count (was entry_count)
bucket_count, # bucket_count
unique_count, # unique_count
value_bits, # value_bits
disp_bits, # disp_bits
prefix_count, # prefix_count
block_size, # block_size (was reserved0)
0, # reserved1
block_size, # block_size
entry_count, # entry_count (was reserved1)
)
assert len(header) == 32, f"Header is {len(header)} bytes, expected 32"
@ -420,7 +376,7 @@ def compile_mphf(entries, max_size=None, block_size=4096):
disp_reader.bit_pos = bucket * disp_bits
d = disp_reader.read_bits(disp_bits)
slot = hash_key(kb, d + 1) % entry_count
slot = hash_key(kb, d + 1) % slot_count
# Check fingerprint
expected_fp = fnv1a_32(kb) & 0xFF
@ -463,12 +419,15 @@ def compile_mphf(entries, max_size=None, block_size=4096):
print(f" Verification passed: all {entry_count} entries OK", file=sys.stderr)
# Check final size
if len(binary) > max_size:
print(f" WARNING: output {len(binary)} bytes exceeds max {max_size}", file=sys.stderr)
# Check final size — hard error, never trim
if max_size and len(binary) > max_size:
print(f"FATAL: output {len(binary)} bytes exceeds budget {max_size}. "
f"Increase budget or split differently.", file=sys.stderr)
return None
return binary, {
'entry_count': entry_count,
'slot_count': slot_count,
'bucket_count': bucket_count,
'unique_count': unique_count,
'value_bits': value_bits,
@ -489,12 +448,12 @@ def compile_mphf(entries, max_size=None, block_size=4096):
def print_stats(stats):
"""Print size breakdown statistics."""
print(f"Entries: {stats['entry_count']}")
print(f"Entries: {stats['entry_count']}, slots: {stats.get('slot_count', stats['entry_count'])}")
print(f"Block size: {stats.get('block_size', 4096)} bytes")
print(f"MPHF displacements: {stats['disp_section_bytes']/1024:.1f} KB "
f"({stats['bucket_count']} buckets, {stats['disp_bits']} bits each)")
print(f"Value array: {stats['val_section_bytes']/1024:.1f} KB "
f"({stats['entry_count']} entries, {stats['value_bits']} bits each)")
f"({stats.get('slot_count', stats['entry_count'])} slots, {stats['value_bits']} bits each)")
print(f"Fingerprints: {stats['fp_section_bytes']/1024:.1f} KB")
print(f"String offsets: {stats['str_offsets_bytes']/1024:.1f} KB "
f"({stats['unique_count']} unique x 3 bytes)")
@ -534,8 +493,6 @@ def main():
parser.add_argument('output', help='Output binary file')
parser.add_argument('--max-size', type=int, default=462*1024,
help='Maximum output size in bytes (default: 473088 = 462KB)')
parser.add_argument('--max-entries', type=int, default=None,
help='Maximum number of entries')
parser.add_argument('--stats', action='store_true',
help='Print size breakdown statistics')
parser.add_argument('--verify', action='store_true', default=True,
@ -558,11 +515,6 @@ def main():
entries = list(raw_dict.items())
if args.max_entries is not None:
entries_scored = sorted(entries, key=lambda e: score_entry(e[0], e[1]))
entries = entries_scored[:args.max_entries]
print(f"Trimmed to {len(entries)} entries (--max-entries)", file=sys.stderr)
# Split-partition mode
if args.split_part:
left_entries, right_entries = partition_entries(