afiqzudinhadi/zmk-steno-engine
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Never trim dictionary — hard error if over budget

Browse source 
Removed all trimming/auto-trim logic. Build fails if compiled dict
exceeds flash budget. CHD now uses spare slots (1.23x load factor)
and fewer buckets (entry_count/5) for reliable construction at any
entry count. Binary format bumped to v3: header stores slot_count
and entry_count separately.
This commit is contained in:
afiqzudinhadi 2026-07-02 15:39:52 +08:00
parent 1339ec4525
commit 28e0c1a49f
4 changed files with 59 additions and 103 deletions
Download patch file Download diff file Expand all files Collapse all files

14
src/dict_mphf.c
View file

@ -158,15 +158,15 @@ int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len)
dict->disp_section_len = align4((disp_bits_total + 7) / 8);
offset += dict->disp_section_len;
/* Values */
/* Values (slot_count slots, not entry_count) */
dict->values = base + offset;
uint32_t val_bits_total = (uint32_t)hdr->entry_count * hdr->value_bits;
uint32_t val_bits_total = (uint32_t)hdr->slot_count * hdr->value_bits;
dict->val_section_len = align4((val_bits_total + 7) / 8);
offset += dict->val_section_len;
/* Fingerprints */
/* Fingerprints (one per slot) */
dict->fingerprints = base + offset;
dict->fp_section_len = align4(hdr->entry_count);
dict->fp_section_len = align4(hdr->slot_count);
offset += dict->fp_section_len;
/* String offsets (u24 LE, 3 bytes each) */
@ -189,7 +189,7 @@ int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len)
}
/* Block size from header (0 = legacy default 4096) */
dict->blk_size = hdr->block_size ? hdr->block_size : DICT_MPHF_BLOCK_SIZE;
dict->blk_size = hdr->block_size ? hdr->block_size : 4096;
/* Prefix table at end */
uint32_t prefix_bytes = (uint32_t)hdr->prefix_count * 4;
@ -236,7 +236,7 @@ static const char *resolve_string(const struct dict_mphf *dict, uint32_t val_id)
const uint8_t *compressed = dict->blocks_start + blk_start;
uint32_t compressed_len = blk_end - blk_start;
static uint8_t decomp_buf[DICT_MPHF_BLOCK_SIZE];
static uint8_t decomp_buf[4096];
static uint32_t cached_block = UINT32_MAX;
static size_t cached_len;
@ -285,7 +285,7 @@ const char *dict_mphf_lookup(const struct dict_mphf *dict,
uint32_t d = read_bits(dict->displacements,
bucket * (uint32_t)hdr->disp_bits,
hdr->disp_bits);
uint32_t slot = hash_key(key_buf, key_len, d + 1) % hdr->entry_count;
uint32_t slot = hash_key(key_buf, key_len, d + 1) % hdr->slot_count;
uint8_t expected_fp = (uint8_t)(fnv1a_32(key_buf, key_len) & 0xFF);
if (dict->fingerprints[slot] != expected_fp) {

20
src/dict_mphf.h
View file

@ -1,8 +1,9 @@
/**
* MPHF (Minimal Perfect Hash Function) dictionary lookup engine.
*
* Binary format v2: CHD MPHF + bit-packed displacements/values +
* fingerprinted verification + block-compressed string table.
* Binary format v3: CHD MPHF with spare slots + bit-packed
* displacements/values + fingerprinted verification +
* block-compressed string table.
*
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*/
@ -15,24 +16,22 @@
#include <stddef.h>
#define DICT_MPHF_MAGIC 0x4F4E5453 /* "STNO" */
#define DICT_MPHF_VERSION 2
#define DICT_MPHF_VERSION 3
#define DICT_MPHF_FLAG_COMPRESSED 0x0001
#define DICT_MPHF_BLOCK_SIZE 4096
struct dict_mphf_header {
uint32_t magic;
uint16_t version;
uint16_t flags;
uint32_t entry_count;
uint32_t slot_count;
uint32_t bucket_count;
uint32_t unique_count;
uint8_t value_bits;
uint8_t disp_bits;
uint16_t prefix_count;
uint32_t block_size; /* zlib block size (0 = default 4096) */
uint32_t reserved1;
uint32_t block_size;
uint32_t entry_count;
} __attribute__((packed));
_Static_assert(sizeof(struct dict_mphf_header) == 32, "header must be 32 bytes");
@ -67,4 +66,9 @@ static inline uint32_t dict_mphf_count(const struct dict_mphf *dict)
return dict->header->entry_count;
}
static inline uint32_t dict_mphf_slot_count(const struct dict_mphf *dict)
{
return dict->header->slot_count;
}
#endif /* DICT_MPHF_H */