afiqzudinhadi/zmk-steno-engine
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Revert to v2 format, fix CHD with more displacement tries

Browse source 
Spare slots (1.23x) doubled binary size → reverted to v2 (no spare
slots, entry_count == slot_count). CHD now uses entry_count/4
buckets and 1M displacement tries instead of 65K. No trimming.
This commit is contained in:
afiqzudinhadi 2026-07-02 15:47:23 +08:00
parent ab0e90b84b
commit 1d5bef3129
3 changed files with 36 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

10
src/dict_mphf.c
View file

@ -158,15 +158,15 @@ int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len)
dict->disp_section_len = align4((disp_bits_total + 7) / 8); dict->disp_section_len = align4((disp_bits_total + 7) / 8);
offset += dict->disp_section_len; offset += dict->disp_section_len;
/* Values (slot_count slots, not entry_count) */ /* Values */
dict->values = base + offset; dict->values = base + offset;
uint32_t val_bits_total = (uint32_t)hdr->slot_count * hdr->value_bits; uint32_t val_bits_total = (uint32_t)hdr->entry_count * hdr->value_bits;
dict->val_section_len = align4((val_bits_total + 7) / 8); dict->val_section_len = align4((val_bits_total + 7) / 8);
offset += dict->val_section_len; offset += dict->val_section_len;
/* Fingerprints (one per slot) */ /* Fingerprints */
dict->fingerprints = base + offset; dict->fingerprints = base + offset;
dict->fp_section_len = align4(hdr->slot_count); dict->fp_section_len = align4(hdr->entry_count);
offset += dict->fp_section_len; offset += dict->fp_section_len;
/* String offsets (u24 LE, 3 bytes each) */ /* String offsets (u24 LE, 3 bytes each) */
@ -285,7 +285,7 @@ const char *dict_mphf_lookup(const struct dict_mphf *dict,
uint32_t d = read_bits(dict->displacements, uint32_t d = read_bits(dict->displacements,
bucket * (uint32_t)hdr->disp_bits, bucket * (uint32_t)hdr->disp_bits,
hdr->disp_bits); hdr->disp_bits);
uint32_t slot = hash_key(key_buf, key_len, d + 1) % hdr->slot_count; uint32_t slot = hash_key(key_buf, key_len, d + 1) % hdr->entry_count;
uint8_t expected_fp = (uint8_t)(fnv1a_32(key_buf, key_len) & 0xFF); uint8_t expected_fp = (uint8_t)(fnv1a_32(key_buf, key_len) & 0xFF);
if (dict->fingerprints[slot] != expected_fp) { if (dict->fingerprints[slot] != expected_fp) {

26
src/dict_mphf.h
View file

@ -1,9 +1,8 @@
/** /**
* MPHF (Minimal Perfect Hash Function) dictionary lookup engine. * MPHF (Minimal Perfect Hash Function) dictionary lookup engine.
* *
* Binary format v3: CHD MPHF with spare slots + bit-packed * Binary format v2: CHD MPHF + bit-packed displacements/values +
* displacements/values + fingerprinted verification + * fingerprinted verification + block-compressed string table.
* block-compressed string table.
* *
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 * SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*/ */
@ -16,7 +15,7 @@
#include <stddef.h> #include <stddef.h>
#define DICT_MPHF_MAGIC 0x4F4E5453 /* "STNO" */ #define DICT_MPHF_MAGIC 0x4F4E5453 /* "STNO" */
#define DICT_MPHF_VERSION 3 #define DICT_MPHF_VERSION 2
#define DICT_MPHF_FLAG_COMPRESSED 0x0001 #define DICT_MPHF_FLAG_COMPRESSED 0x0001
@ -24,14 +23,14 @@ struct dict_mphf_header {
uint32_t magic; uint32_t magic;
uint16_t version; uint16_t version;
uint16_t flags; uint16_t flags;
uint32_t slot_count; uint32_t entry_count;
uint32_t bucket_count; uint32_t bucket_count;
uint32_t unique_count; uint32_t unique_count;
uint8_t value_bits; uint8_t value_bits;
uint8_t disp_bits; uint8_t disp_bits;
uint16_t prefix_count; uint16_t prefix_count;
uint32_t block_size; uint32_t block_size;
uint32_t entry_count; uint32_t reserved1;
} __attribute__((packed)); } __attribute__((packed));
_Static_assert(sizeof(struct dict_mphf_header) == 32, "header must be 32 bytes"); _Static_assert(sizeof(struct dict_mphf_header) == 32, "header must be 32 bytes");
@ -42,16 +41,16 @@ struct dict_mphf {
const uint8_t *values; const uint8_t *values;
const uint8_t *fingerprints; const uint8_t *fingerprints;
const uint8_t *string_offsets; const uint8_t *string_offsets;
const uint8_t *str_data_start; /* start of string data section */ const uint8_t *str_data_start;
const uint32_t *prefix_table; const uint32_t *prefix_table;
uint32_t disp_section_len; uint32_t disp_section_len;
uint32_t val_section_len; uint32_t val_section_len;
uint32_t fp_section_len; uint32_t fp_section_len;
uint16_t block_count; uint16_t block_count;
const uint32_t *block_dir; /* block offset directory */ const uint32_t *block_dir;
const uint8_t *blocks_start; /* start of compressed blocks */ const uint8_t *blocks_start;
uint32_t str_data_len; /* total string data section length */ uint32_t str_data_len;
uint32_t blk_size; /* actual block size from header */ uint32_t blk_size;
}; };
int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len); int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len);
@ -66,9 +65,4 @@ static inline uint32_t dict_mphf_count(const struct dict_mphf *dict)
return dict->header->entry_count; return dict->header->entry_count;
} }
static inline uint32_t dict_mphf_slot_count(const struct dict_mphf *dict)
{
return dict->header->slot_count;
}
#endif /* DICT_MPHF_H */ #endif /* DICT_MPHF_H */

45
tools/compile_mphf.py
View file

@ -166,13 +166,11 @@ def build_chd(keys_and_bytes, entry_count):
keys_and_bytes: list of (index, key_bytes) for each entry keys_and_bytes: list of (index, key_bytes) for each entry
entry_count: total number of entries entry_count: total number of entries
Returns: (displacements, slot_to_entry_idx, slot_count, max_displacement) Returns: (displacements, slot_to_entry_idx, max_displacement)
displacements[bucket] = d value displacements[bucket] = d value
slot_to_entry_idx[slot] = index into keys_and_bytes, or -1 if empty slot_to_entry_idx[slot] = index into keys_and_bytes, or -1 if empty
slot_count: actual number of slots (>= entry_count)
""" """
bucket_count = max(entry_count // 5, 16) bucket_count = max(entry_count // 4, 16)
slot_count = math.ceil(entry_count * 1.23)
# Assign keys to buckets # Assign keys to buckets
buckets = defaultdict(list) buckets = defaultdict(list)
@ -185,7 +183,7 @@ def build_chd(keys_and_bytes, entry_count):
displacements = [0] * bucket_count displacements = [0] * bucket_count
occupied = set() occupied = set()
slot_to_entry = [-1] * slot_count slot_to_entry = [-1] * entry_count
max_disp = 0 max_disp = 0
for bucket_id, members in sorted_buckets: for bucket_id, members in sorted_buckets:
@ -195,13 +193,13 @@ def build_chd(keys_and_bytes, entry_count):
member_key_bytes = [(m, keys_and_bytes[m][1]) for m in members] member_key_bytes = [(m, keys_and_bytes[m][1]) for m in members]
placed = False placed = False
for d in range(65536): for d in range(1 << 20):
slots = [] slots = []
collision = False collision = False
seen = set() seen = set()
for _, kb in member_key_bytes: for _, kb in member_key_bytes:
slot = hash_key(kb, d + 1) % slot_count slot = hash_key(kb, d + 1) % entry_count
if slot in occupied or slot in seen: if slot in occupied or slot in seen:
collision = True collision = True
break break
@ -223,11 +221,11 @@ def build_chd(keys_and_bytes, entry_count):
break break
if not placed: if not placed:
print(f"FATAL: bucket {bucket_id} with {len(members)} keys failed after 65536 tries", print(f"FATAL: bucket {bucket_id} with {len(members)} keys failed after 1048576 tries",
file=sys.stderr) file=sys.stderr)
return None, None, None, None return None, None, None
return displacements, slot_to_entry, slot_count, max_disp return displacements, slot_to_entry, max_disp
# ─── Compilation ─── # ─── Compilation ───
@ -294,7 +292,7 @@ def compile_mphf(entries, max_size=None, block_size=4096):
# Build CHD # Build CHD
chd_input = [(i, keys_and_bytes[i][0]) for i in range(entry_count)] chd_input = [(i, keys_and_bytes[i][0]) for i in range(entry_count)]
displacements, slot_to_entry, slot_count, max_disp = build_chd(chd_input, entry_count) displacements, slot_to_entry, max_disp = build_chd(chd_input, entry_count)
if displacements is None: if displacements is None:
return None return None
@ -306,7 +304,7 @@ def compile_mphf(entries, max_size=None, block_size=4096):
value_bits = max(1, math.ceil(math.log2(max(unique_count, 2)))) value_bits = max(1, math.ceil(math.log2(max(unique_count, 2))))
prefix_count = len(prefix_list) prefix_count = len(prefix_list)
print(f" Slots: {slot_count}, buckets: {bucket_count}", file=sys.stderr) print(f" Buckets: {bucket_count}", file=sys.stderr)
print(f" Max displacement: {max_disp}, disp_bits: {disp_bits}", file=sys.stderr) print(f" Max displacement: {max_disp}, disp_bits: {disp_bits}", file=sys.stderr)
print(f" Unique translations: {unique_count}, value_bits: {value_bits}", file=sys.stderr) print(f" Unique translations: {unique_count}, value_bits: {value_bits}", file=sys.stderr)
print(f" Prefix entries: {prefix_count}", file=sys.stderr) print(f" Prefix entries: {prefix_count}", file=sys.stderr)
@ -320,10 +318,10 @@ def compile_mphf(entries, max_size=None, block_size=4096):
disp_writer.pad_to_alignment(4) disp_writer.pad_to_alignment(4)
disp_section = disp_writer.to_bytes() disp_section = disp_writer.to_bytes()
# Values section: slot → value_id (slot_count slots, not entry_count) # Values section: slot → value_id
val_writer = BitWriter() val_writer = BitWriter()
fingerprints = bytearray(slot_count) fingerprints = bytearray(entry_count)
for slot in range(slot_count): for slot in range(entry_count):
entry_idx = slot_to_entry[slot] entry_idx = slot_to_entry[slot]
if entry_idx >= 0: if entry_idx >= 0:
kb, trans, stroke_str, strokes = keys_and_bytes[entry_idx] kb, trans, stroke_str, strokes = keys_and_bytes[entry_idx]
@ -358,21 +356,21 @@ def compile_mphf(entries, max_size=None, block_size=4096):
# Header (32 bytes): # Header (32 bytes):
# magic: u32, version: u16, flags: u16, # magic: u32, version: u16, flags: u16,
# slot_count: u32, bucket_count: u32, unique_count: u32, # entry_count: u32, bucket_count: u32, unique_count: u32,
# value_bits: u8, disp_bits: u8, prefix_count: u16, # value_bits: u8, disp_bits: u8, prefix_count: u16,
# block_size: u32, entry_count: u32 # block_size: u32, reserved1: u32
header = struct.pack('<IHHIIIBBHii', header = struct.pack('<IHHIIIBBHii',
0x4F4E5453, # magic "STNO" 0x4F4E5453, # magic "STNO"
3, # version (bumped: slot_count in header) 2, # version
0x0001, # flags: bit 0 = block-compressed strings 0x0001, # flags: bit 0 = block-compressed strings
slot_count, # slot_count (was entry_count) entry_count, # entry_count
bucket_count, # bucket_count bucket_count, # bucket_count
unique_count, # unique_count unique_count, # unique_count
value_bits, # value_bits value_bits, # value_bits
disp_bits, # disp_bits disp_bits, # disp_bits
prefix_count, # prefix_count prefix_count, # prefix_count
block_size, # block_size block_size, # block_size
entry_count, # entry_count (was reserved1) 0, # reserved1
) )
assert len(header) == 32, f"Header is {len(header)} bytes, expected 32" assert len(header) == 32, f"Header is {len(header)} bytes, expected 32"
@ -392,7 +390,7 @@ def compile_mphf(entries, max_size=None, block_size=4096):
disp_reader.bit_pos = bucket * disp_bits disp_reader.bit_pos = bucket * disp_bits
d = disp_reader.read_bits(disp_bits) d = disp_reader.read_bits(disp_bits)
slot = hash_key(kb, d + 1) % slot_count slot = hash_key(kb, d + 1) % entry_count
# Check fingerprint # Check fingerprint
expected_fp = fnv1a_32(kb) & 0xFF expected_fp = fnv1a_32(kb) & 0xFF
@ -443,7 +441,6 @@ def compile_mphf(entries, max_size=None, block_size=4096):
return binary, { return binary, {
'entry_count': entry_count, 'entry_count': entry_count,
'slot_count': slot_count,
'bucket_count': bucket_count, 'bucket_count': bucket_count,
'unique_count': unique_count, 'unique_count': unique_count,
'value_bits': value_bits, 'value_bits': value_bits,
@ -464,12 +461,12 @@ def compile_mphf(entries, max_size=None, block_size=4096):
def print_stats(stats): def print_stats(stats):
"""Print size breakdown statistics.""" """Print size breakdown statistics."""
print(f"Entries: {stats['entry_count']}, slots: {stats.get('slot_count', stats['entry_count'])}") print(f"Entries: {stats['entry_count']}")
print(f"Block size: {stats.get('block_size', 4096)} bytes") print(f"Block size: {stats.get('block_size', 4096)} bytes")
print(f"MPHF displacements: {stats['disp_section_bytes']/1024:.1f} KB " print(f"MPHF displacements: {stats['disp_section_bytes']/1024:.1f} KB "
f"({stats['bucket_count']} buckets, {stats['disp_bits']} bits each)") f"({stats['bucket_count']} buckets, {stats['disp_bits']} bits each)")
print(f"Value array: {stats['val_section_bytes']/1024:.1f} KB " print(f"Value array: {stats['val_section_bytes']/1024:.1f} KB "
f"({stats.get('slot_count', stats['entry_count'])} slots, {stats['value_bits']} bits each)") f"({stats['entry_count']} entries, {stats['value_bits']} bits each)")
print(f"Fingerprints: {stats['fp_section_bytes']/1024:.1f} KB") print(f"Fingerprints: {stats['fp_section_bytes']/1024:.1f} KB")
print(f"String offsets: {stats['str_offsets_bytes']/1024:.1f} KB " print(f"String offsets: {stats['str_offsets_bytes']/1024:.1f} KB "
f"({stats['unique_count']} unique x 3 bytes)") f"({stats['unique_count']} unique x 3 bytes)")