From 20218fa2abe5f9b5efb2cca6975774cd25e98ecd Mon Sep 17 00:00:00 2001 From: afiqzudinhadi Date: Thu, 2 Jul 2026 11:08:28 +0800 Subject: [PATCH] True split-dict: partition dictionary across both halves MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Importance-based partitioning — highest-importance entries on left (central) for zero-latency local lookup, remainder on right (peripheral) queried over BLE on miss. Both halves embed their own MPHF binary. Configurable block size for tighter compression. --- CMakeLists.txt | 72 ++++++++++++++++++++++++++++--------------- Kconfig | 28 ++++++++++++++--- src/behavior_steno.c | 31 +++++++++++++++++-- src/dict_mphf.c | 7 +++-- src/dict_mphf.h | 3 +- src/split_dict.c | 56 +++++++++++++++++++++++---------- tools/compile_mphf.py | 70 ++++++++++++++++++++++++++++++++++++----- 7 files changed, 209 insertions(+), 58 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0d59f82..24e7090 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,30 +9,31 @@ target_include_directories(app PRIVATE ) # ── Split-dict mode ────────────────────────────────── -# Central: behavior engine + BLE dict client -# Peripheral: dict embed + lookup engine + GATT server +# TRUE split: both halves have local MPHF partition + BLE fallback +# Central: behavior engine + local left partition + BLE client +# Peripheral: local right partition + GATT server if(CONFIG_STENO_SPLIT_DICT) if(CONFIG_ZMK_SPLIT_ROLE_CENTRAL) - # Central side: behavior engine queries peripheral over BLE + # Central: behavior engine + local dict + BLE client for remote partition target_sources(app PRIVATE src/behavior_steno.c src/output.c src/formatter.c src/undo.c + src/dict_embed.S + src/dict_mphf.c src/split_dict.c src/split_cache.c ) else() - # Peripheral side: dict embedded here, serves GATT queries + # Peripheral: local dict + GATT server for central queries target_sources(app PRIVATE src/dict_embed.S + src/dict_mphf.c + src/split_dict.c + src/split_cache.c ) - if(CONFIG_STENO_DICT_MPHF) - target_sources(app PRIVATE src/dict_mphf.c) - else() - target_sources(app PRIVATE src/trie.c) - endif() endif() # ── Non-split mode ─────────────────────────────────── @@ -55,12 +56,11 @@ else() endif() # ── Dictionary compilation ─────────────────────────── -# Build dict binary when we embed it (non-split, or split peripheral) +# Both halves need dict embed in split mode; non-split same as before set(STENO_NEED_DICT_EMBED FALSE) if(CONFIG_STENO_SPLIT_DICT) - if(NOT CONFIG_ZMK_SPLIT_ROLE_CENTRAL) - set(STENO_NEED_DICT_EMBED TRUE) - endif() + # Both central and peripheral need their own partition + set(STENO_NEED_DICT_EMBED TRUE) elseif(NOT CONFIG_ZMK_SPLIT OR CONFIG_ZMK_SPLIT_ROLE_CENTRAL) set(STENO_NEED_DICT_EMBED TRUE) endif() @@ -96,19 +96,43 @@ if(STENO_NEED_DICT_EMBED) # Compile dict binary if(EXISTS ${STENO_DICT_SRC}) - if(CONFIG_STENO_DICT_MPHF) - # Fetch at build time if hash changed - if(DEFINED STENO_DICT_NAME) - add_custom_command( - OUTPUT ${STENO_DICT_SRC}.stamp - COMMAND ${Python3_EXECUTABLE} ${STENO_FETCH} - ${STENO_DICT_NAME} ${STENO_DICTS_DIR} - COMMAND ${CMAKE_COMMAND} -E touch ${STENO_DICT_SRC}.stamp - COMMENT "Checking ${STENO_DICT_NAME} dictionary for updates" - ) - add_custom_target(steno_dict_fetch DEPENDS ${STENO_DICT_SRC}.stamp) + # Fetch at build time if hash changed + if(DEFINED STENO_DICT_NAME) + add_custom_command( + OUTPUT ${STENO_DICT_SRC}.stamp + COMMAND ${Python3_EXECUTABLE} ${STENO_FETCH} + ${STENO_DICT_NAME} ${STENO_DICTS_DIR} + COMMAND ${CMAKE_COMMAND} -E touch ${STENO_DICT_SRC}.stamp + COMMENT "Checking ${STENO_DICT_NAME} dictionary for updates" + ) + add_custom_target(steno_dict_fetch DEPENDS ${STENO_DICT_SRC}.stamp) + endif() + + if(CONFIG_STENO_SPLIT_DICT) + # Split mode: compile the appropriate partition + if(CONFIG_ZMK_SPLIT_ROLE_CENTRAL) + set(STENO_SPLIT_PART "left") + set(STENO_SPLIT_SIZE ${CONFIG_STENO_DICT_LEFT_MAX_SIZE}) + else() + set(STENO_SPLIT_PART "right") + set(STENO_SPLIT_SIZE ${CONFIG_STENO_DICT_RIGHT_MAX_SIZE}) endif() + add_custom_command( + OUTPUT ${STENO_DICT_BIN} + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_mphf.py + ${STENO_DICT_SRC} ${STENO_DICT_BIN} + --split-part ${STENO_SPLIT_PART} + --left-size ${CONFIG_STENO_DICT_LEFT_MAX_SIZE} + --right-size ${CONFIG_STENO_DICT_RIGHT_MAX_SIZE} + --max-size ${STENO_SPLIT_SIZE} + --block-size 2048 + DEPENDS ${STENO_DICT_SRC} + ${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_mphf.py + COMMENT "Compiling steno dictionary (${STENO_SPLIT_PART} partition)" + ) + elseif(CONFIG_STENO_DICT_MPHF) add_custom_command( OUTPUT ${STENO_DICT_BIN} COMMAND ${Python3_EXECUTABLE} diff --git a/Kconfig b/Kconfig index e66b5c4..6d5cec8 100644 --- a/Kconfig +++ b/Kconfig @@ -81,15 +81,31 @@ config STENO_DICT_MAX_SIZE The MPHF compiler auto-trims to fit. menuconfig STENO_SPLIT_DICT - bool "Split dictionary storage on peripheral" + bool "Split dictionary across both halves" default n + select STENO_DICT_MPHF help - Store the steno dictionary on the peripheral half and - perform lookups over BLE. This frees flash on the central - side at the cost of added lookup latency. + Partition the steno dictionary across both keyboard halves. + Left (central) gets highest-importance entries for zero-latency + local lookup. Right (peripheral) gets remaining entries, queried + over BLE on local miss. Both halves have their own MPHF dict. if STENO_SPLIT_DICT +config STENO_DICT_LEFT_MAX_SIZE + int "Left half dict budget (bytes)" + default 430080 + help + Flash budget for left (central) partition. Default 420KB. + Highest-importance entries fill this first. + +config STENO_DICT_RIGHT_MAX_SIZE + int "Right half dict budget (bytes)" + default 545792 + help + Flash budget for right (peripheral) partition. Default 533KB. + Remaining entries after left partition is filled. + config STENO_SPLIT_CACHE_SIZE int "LRU cache entries on central side" default 64 @@ -104,6 +120,10 @@ config STENO_SPLIT_TIMEOUT_MS default 50 range 10 500 +config STENO_SPLIT_LOG_LEVEL + int "Split dict log level" + default 3 + endif # STENO_SPLIT_DICT endif # STENO_ENGINE diff --git a/src/behavior_steno.c b/src/behavior_steno.c index adfa1af..25599ee 100644 --- a/src/behavior_steno.c +++ b/src/behavior_steno.c @@ -18,6 +18,7 @@ #if IS_ENABLED(CONFIG_STENO_SPLIT_DICT) #include "split_dict.h" +#include "dict_mphf.h" #elif IS_ENABLED(CONFIG_STENO_DICT_MPHF) #include "dict_mphf.h" #else @@ -26,12 +27,10 @@ LOG_MODULE_DECLARE(zmk, CONFIG_ZMK_LOG_LEVEL); -#if !IS_ENABLED(CONFIG_STENO_SPLIT_DICT) extern const uint8_t _steno_dict_start[]; extern const uint8_t _steno_dict_end[]; -#endif -#if IS_ENABLED(CONFIG_STENO_DICT_MPHF) && !IS_ENABLED(CONFIG_STENO_SPLIT_DICT) +#if IS_ENABLED(CONFIG_STENO_DICT_MPHF) || IS_ENABLED(CONFIG_STENO_SPLIT_DICT) static struct dict_mphf mphf_dict; #endif @@ -57,6 +56,12 @@ static void multi_timeout_handler(struct k_work *work); static const char *do_lookup(const uint32_t *strokes, uint8_t count) { #if IS_ENABLED(CONFIG_STENO_SPLIT_DICT) + /* Try local partition first (zero latency) */ + const char *local = dict_mphf_lookup(&mphf_dict, strokes, count); + if (local) { + return local; + } + /* Miss → query remote partition over BLE */ static char split_buf[128]; int ret = split_dict_lookup(strokes, count, split_buf, sizeof(split_buf)); return (ret > 0) ? split_buf : NULL; @@ -70,6 +75,11 @@ static const char *do_lookup(const uint32_t *strokes, uint8_t count) static bool do_has_prefix(const uint32_t *strokes, uint8_t count) { #if IS_ENABLED(CONFIG_STENO_SPLIT_DICT) + /* Check local prefix table first */ + if (count == 1 && dict_mphf_has_prefix(&mphf_dict, strokes[0])) { + return true; + } + /* Fall through to remote */ return split_dict_has_prefix(strokes, count); #elif IS_ENABLED(CONFIG_STENO_DICT_MPHF) return (count == 1) ? dict_mphf_has_prefix(&mphf_dict, strokes[0]) : false; @@ -225,6 +235,21 @@ static int behavior_steno_init(const struct device *dev) k_work_init_delayable(&state.multi_timeout, multi_timeout_handler); #if IS_ENABLED(CONFIG_STENO_SPLIT_DICT) + /* Init local partition dict */ + { + size_t dict_size = _steno_dict_end - _steno_dict_start; + if (dict_size > 4) { + int ret = dict_mphf_init(&mphf_dict, _steno_dict_start, dict_size); + if (ret == 0) { + LOG_INF("Local partition loaded (%u bytes)", (unsigned)dict_size); + } else { + LOG_ERR("Local partition init failed: %d", ret); + } + } else { + LOG_WRN("No local partition embedded"); + } + } + /* Init BLE client for remote partition */ split_dict_init(); dict_ready = true; #else diff --git a/src/dict_mphf.c b/src/dict_mphf.c index a4463ea..8151917 100644 --- a/src/dict_mphf.c +++ b/src/dict_mphf.c @@ -188,6 +188,9 @@ int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len) dict->blocks_start = dict->str_data_start; } + /* Block size from header (0 = legacy default 4096) */ + dict->blk_size = hdr->block_size ? hdr->block_size : DICT_MPHF_BLOCK_SIZE; + /* Prefix table at end */ uint32_t prefix_bytes = (uint32_t)hdr->prefix_count * 4; if (len >= prefix_bytes) { @@ -213,8 +216,8 @@ static const char *resolve_string(const struct dict_mphf *dict, uint32_t val_id) } /* Block-compressed: decompress the right block */ - uint32_t block_idx = str_offset / DICT_MPHF_BLOCK_SIZE; - uint32_t in_block_off = str_offset % DICT_MPHF_BLOCK_SIZE; + uint32_t block_idx = str_offset / dict->blk_size; + uint32_t in_block_off = str_offset % dict->blk_size; if (block_idx >= dict->block_count) { return NULL; diff --git a/src/dict_mphf.h b/src/dict_mphf.h index d78f41b..f5f859e 100644 --- a/src/dict_mphf.h +++ b/src/dict_mphf.h @@ -31,7 +31,7 @@ struct dict_mphf_header { uint8_t value_bits; uint8_t disp_bits; uint16_t prefix_count; - uint32_t reserved0; + uint32_t block_size; /* zlib block size (0 = default 4096) */ uint32_t reserved1; } __attribute__((packed)); @@ -52,6 +52,7 @@ struct dict_mphf { const uint32_t *block_dir; /* block offset directory */ const uint8_t *blocks_start; /* start of compressed blocks */ uint32_t str_data_len; /* total string data section length */ + uint32_t blk_size; /* actual block size from header */ }; int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len); diff --git a/src/split_dict.c b/src/split_dict.c index 45bfb91..580ddbb 100644 --- a/src/split_dict.c +++ b/src/split_dict.c @@ -18,6 +18,7 @@ #include "split_dict.h" #include "split_cache.h" +#include "dict_mphf.h" LOG_MODULE_REGISTER(split_dict, CONFIG_STENO_SPLIT_LOG_LEVEL); @@ -33,10 +34,11 @@ static uint8_t seq_counter; /* Cache instance */ static struct split_cache dict_cache; -/* External trie lookup (peripheral side) */ -extern int trie_lookup(const uint32_t *strokes, uint8_t count, - char *result, size_t result_size); -extern bool trie_has_prefix(const uint32_t *strokes, uint8_t count); +/* Local MPHF dict for peripheral-side GATT lookups */ +extern const uint8_t _steno_dict_start[]; +extern const uint8_t _steno_dict_end[]; +static struct dict_mphf peripheral_mphf; +static bool peripheral_dict_ready; /* --- Helpers --- */ @@ -91,14 +93,17 @@ static ssize_t dict_query_write_cb(struct bt_conn *conn, resp->msg_type = STENO_MSG_RESPONSE; resp->seq = pkt->seq; - char translation[128]; - int ret = trie_lookup(strokes, stroke_count, translation, sizeof(translation)); + const char *translation = NULL; + if (peripheral_dict_ready) { + translation = dict_mphf_lookup(&peripheral_mphf, strokes, stroke_count); + } - if (ret > 0) { + if (translation) { + uint16_t tlen = (uint16_t)strlen(translation); resp->status = STENO_STATUS_FOUND; - resp->data_len = (uint16_t)ret; - memcpy(resp->data, translation, ret); - response_len = sizeof(struct steno_response_pkt) + ret; + resp->data_len = tlen; + memcpy(resp->data, translation, tlen); + response_len = sizeof(struct steno_response_pkt) + tlen; } else { resp->status = STENO_STATUS_NOT_FOUND; resp->data_len = 0; @@ -135,7 +140,9 @@ static ssize_t dict_prefix_write_cb(struct bt_conn *conn, resp->seq = pkt->seq; resp->data_len = 0; - if (trie_has_prefix(strokes, stroke_count)) { + if (peripheral_dict_ready && + stroke_count == 1 && + dict_mphf_has_prefix(&peripheral_mphf, strokes[0])) { resp->status = STENO_STATUS_PREFIX_ONLY; } else { resp->status = STENO_STATUS_NOT_FOUND; @@ -188,14 +195,17 @@ static ssize_t dict_batch_write_cb(struct bt_conn *conn, resp->msg_type = STENO_MSG_RESPONSE; resp->seq = pkt->seq; - char translation[128]; - int ret = trie_lookup(strokes, stroke_count, translation, sizeof(translation)); + const char *translation = NULL; + if (peripheral_dict_ready) { + translation = dict_mphf_lookup(&peripheral_mphf, strokes, stroke_count); + } - if (ret > 0) { + if (translation) { + uint16_t tlen = (uint16_t)strlen(translation); resp->status = STENO_STATUS_FOUND; - resp->data_len = (uint16_t)ret; - memcpy(resp->data, translation, ret); - response_len = sizeof(struct steno_response_pkt) + ret; + resp->data_len = tlen; + memcpy(resp->data, translation, tlen); + response_len = sizeof(struct steno_response_pkt) + tlen; } else { resp->status = STENO_STATUS_NOT_FOUND; resp->data_len = 0; @@ -453,6 +463,18 @@ int split_dict_init(void) seq_counter = 0; split_conn = NULL; + /* Init peripheral-side MPHF dict for GATT lookups */ + size_t dict_size = _steno_dict_end - _steno_dict_start; + if (dict_size > 4) { + int ret = dict_mphf_init(&peripheral_mphf, _steno_dict_start, dict_size); + if (ret == 0) { + peripheral_dict_ready = true; + LOG_INF("Peripheral partition loaded (%u bytes)", (unsigned)dict_size); + } else { + LOG_ERR("Peripheral partition init failed: %d", ret); + } + } + LOG_INF("Split dict initialized"); return 0; } diff --git a/tools/compile_mphf.py b/tools/compile_mphf.py index 103e81c..cda1f4d 100755 --- a/tools/compile_mphf.py +++ b/tools/compile_mphf.py @@ -230,10 +230,11 @@ def build_chd(keys_and_bytes, entry_count): # ─── Compilation ─── -def compile_mphf(entries, max_size=None): +def compile_mphf(entries, max_size=None, block_size=4096): """ entries: list of (stroke_str, translation) from JSON dict max_size: max output size in bytes (default: 462*1024 = 473088) + block_size: zlib compression block size (smaller = better ratio, more blocks) Returns: bytes (the compiled binary) or None if can't fit """ @@ -286,7 +287,6 @@ def compile_mphf(entries, max_size=None): string_offsets.append(len(string_data_raw)) string_data_raw += t.encode('utf-8') + b'\x00' - block_size = 4096 compressed_blocks = [] for i in range(0, len(string_data_raw), block_size): block = string_data_raw[i:i + block_size] @@ -388,7 +388,7 @@ def compile_mphf(entries, max_size=None): # magic: u32, version: u16, flags: u16, # entry_count: u32, bucket_count: u32, unique_count: u32, # value_bits: u8, disp_bits: u8, prefix_count: u16, - # reserved0: u32, reserved1: u32 + # block_size: u32, reserved1: u32 header = struct.pack('