diff --git a/CMakeLists.txt b/CMakeLists.txt index 24e7090..e47f80e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -8,108 +8,179 @@ target_include_directories(app PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src ) -# ── Split-dict mode ────────────────────────────────── -# TRUE split: both halves have local MPHF partition + BLE fallback -# Central: behavior engine + local left partition + BLE client -# Peripheral: local right partition + GATT server -if(CONFIG_STENO_SPLIT_DICT) +find_package(Python3 REQUIRED COMPONENTS Interpreter) - if(CONFIG_ZMK_SPLIT_ROLE_CENTRAL) - # Central: behavior engine + local dict + BLE client for remote partition - target_sources(app PRIVATE - src/behavior_steno.c - src/output.c - src/formatter.c - src/undo.c - src/dict_embed.S - src/dict_mphf.c - src/split_dict.c - src/split_cache.c - ) - else() - # Peripheral: local dict + GATT server for central queries - target_sources(app PRIVATE - src/dict_embed.S - src/dict_mphf.c - src/split_dict.c - src/split_cache.c - ) - endif() +set(STENO_DICTS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/dicts) +set(STENO_FETCH ${CMAKE_CURRENT_SOURCE_DIR}/tools/fetch_dict.py) -# ── Non-split mode ─────────────────────────────────── -# Everything on one board (or central-only without split) +# ── Dictionary selection ───────────────────────────── +# Maps the Kconfig choice onto the compiler's --dicts argument. +if(CONFIG_STENO_DICT_PLOVER) + set(STENO_DICTS_ARG "plover") +elseif(CONFIG_STENO_DICT_LAPWING) + set(STENO_DICTS_ARG "lapwing") else() - if(NOT CONFIG_ZMK_SPLIT OR CONFIG_ZMK_SPLIT_ROLE_CENTRAL) - target_sources(app PRIVATE - src/behavior_steno.c - src/output.c - src/formatter.c - src/undo.c - src/dict_embed.S + set(STENO_DICTS_ARG "both") +endif() + +# Ensure a dictionary JSON is present: download at configure time if +# missing (hard error on failure), and add a build-time stamp rule that +# re-checks the upstream hash on fresh build directories. +function(steno_fetch_dict name json_path) + if(NOT EXISTS ${json_path}) + message(STATUS "Steno: downloading ${name} dictionary...") + execute_process( + COMMAND ${Python3_EXECUTABLE} ${STENO_FETCH} ${name} ${STENO_DICTS_DIR} + RESULT_VARIABLE FETCH_RESULT ) - if(CONFIG_STENO_DICT_MPHF) - target_sources(app PRIVATE src/dict_mphf.c) - else() - target_sources(app PRIVATE src/trie.c) + if(NOT FETCH_RESULT EQUAL 0 OR NOT EXISTS ${json_path}) + message(FATAL_ERROR + "Steno: ${name} dictionary is missing and the download failed " + "(expected ${json_path}).") endif() endif() -endif() + add_custom_command( + OUTPUT ${json_path}.stamp + COMMAND ${Python3_EXECUTABLE} ${STENO_FETCH} ${name} ${STENO_DICTS_DIR} + COMMAND ${CMAKE_COMMAND} -E touch ${json_path}.stamp + COMMENT "Checking ${name} dictionary for updates" + ) +endfunction() -# ── Dictionary compilation ─────────────────────────── -# Both halves need dict embed in split mode; non-split same as before -set(STENO_NEED_DICT_EMBED FALSE) -if(CONFIG_STENO_SPLIT_DICT) - # Both central and peripheral need their own partition - set(STENO_NEED_DICT_EMBED TRUE) -elseif(NOT CONFIG_ZMK_SPLIT OR CONFIG_ZMK_SPLIT_ROLE_CENTRAL) - set(STENO_NEED_DICT_EMBED TRUE) -endif() +# ── v4 union split-section format ──────────────────── +# One compiler run emits BOTH half blobs (docs/FORMAT_V4.md §5). +# Left (central): DISP, MEMBERSHIP, FP, CONFLICTS, VALIDX slice [0, k). +# Right (peripheral): STRDIR, STRINGS, CONFLICTS, VALIDX slice [k, n). +if(CONFIG_STENO_DICT_V4) -if(STENO_NEED_DICT_EMBED) - find_package(Python3 REQUIRED COMPONENTS Interpreter) - set(STENO_DICT_BIN ${CMAKE_CURRENT_BINARY_DIR}/steno_dict.bin) - set(STENO_DICTS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/dicts) - set(STENO_FETCH ${CMAKE_CURRENT_SOURCE_DIR}/tools/fetch_dict.py) + if(NOT CONFIG_STENO_SPLIT_DICT) + message(FATAL_ERROR + "STENO_DICT_V4 requires STENO_SPLIT_DICT: the v4 format spans both " + "keyboard halves (left = lookup structures, right = string table). " + "Enable STENO_SPLIT_DICT, or disable STENO_DICT_V4 to fall back to " + "the legacy MPHF format.") + endif() + + set(STENO_V4_LEFT_BIN ${CMAKE_CURRENT_BINARY_DIR}/steno_v4_left.bin) + set(STENO_V4_RIGHT_BIN ${CMAKE_CURRENT_BINARY_DIR}/steno_v4_right.bin) + + if(CONFIG_ZMK_SPLIT_ROLE_CENTRAL) + # Central: behavior engine + left half blob + BLE client + target_sources(app PRIVATE + src/behavior_steno.c + src/output.c + src/formatter.c + src/undo.c + src/dict_v4.c + src/split_dict.c + src/split_cache.c + src/dict_embed.S + ) + set(STENO_DICT_BIN ${STENO_V4_LEFT_BIN}) + else() + # Peripheral: right half blob + GATT server + target_sources(app PRIVATE + src/dict_v4.c + src/split_dict.c + src/split_cache.c + src/dict_embed.S + ) + set(STENO_DICT_BIN ${STENO_V4_RIGHT_BIN}) + endif() + + # The union format is always compiled from both source dicts; + # --dicts controls which of them are included (dicts_mask). + set(STENO_PLOVER_JSON ${STENO_DICTS_DIR}/plover-main.json) + set(STENO_LAPWING_JSON ${STENO_DICTS_DIR}/lapwing.json) + steno_fetch_dict(plover ${STENO_PLOVER_JSON}) + steno_fetch_dict(lapwing ${STENO_LAPWING_JSON}) + + # Single run produces both blobs. The compiler FAILS the build if + # either half exceeds its budget — entries are never trimmed. + add_custom_command( + OUTPUT ${STENO_V4_LEFT_BIN} ${STENO_V4_RIGHT_BIN} + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_v4.py + --plover ${STENO_PLOVER_JSON} --lapwing ${STENO_LAPWING_JSON} + --dicts ${STENO_DICTS_ARG} + --out-dir ${CMAKE_CURRENT_BINARY_DIR} + --left-size ${CONFIG_STENO_DICT_LEFT_MAX_SIZE} + --right-size ${CONFIG_STENO_DICT_RIGHT_MAX_SIZE} + DEPENDS + ${STENO_PLOVER_JSON} ${STENO_LAPWING_JSON} + ${STENO_PLOVER_JSON}.stamp ${STENO_LAPWING_JSON}.stamp + ${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_v4.py + COMMENT "Compiling steno v4 union dictionary (left + right half blobs)" + ) + + add_custom_target(steno_dict_gen DEPENDS ${STENO_DICT_BIN}) + add_dependencies(app steno_dict_gen) + +# ── Legacy MPHF format (STENO_DICT_V4=n fallback) ──── +else() + + if(CONFIG_STENO_DICT_BOTH) + message(FATAL_ERROR + "STENO_DICT_BOTH requires STENO_DICT_V4: the legacy MPHF format " + "holds a single dictionary. Select STENO_DICT_PLOVER or " + "STENO_DICT_LAPWING, or re-enable STENO_DICT_V4.") + endif() if(CONFIG_STENO_DICT_PLOVER) set(STENO_DICT_NAME "plover") set(STENO_DICT_SRC ${STENO_DICTS_DIR}/plover-main.json) - elseif(CONFIG_STENO_DICT_LAPWING) + else() set(STENO_DICT_NAME "lapwing") set(STENO_DICT_SRC ${STENO_DICTS_DIR}/lapwing.json) - else() - set(STENO_DICT_SRC ${STENO_DICTS_DIR}/test.json) endif() - # Auto-download dict if needed (Plover/Lapwing only) - if(DEFINED STENO_DICT_NAME AND NOT EXISTS ${STENO_DICT_SRC}) - message(STATUS "Steno: downloading ${STENO_DICT_NAME} dictionary...") - execute_process( - COMMAND ${Python3_EXECUTABLE} ${STENO_FETCH} - ${STENO_DICT_NAME} ${STENO_DICTS_DIR} - RESULT_VARIABLE FETCH_RESULT - ) - if(NOT FETCH_RESULT EQUAL 0) - message(WARNING "Steno: dict download failed. Build may fail.") - endif() - endif() - - # Compile dict binary - if(EXISTS ${STENO_DICT_SRC}) - # Fetch at build time if hash changed - if(DEFINED STENO_DICT_NAME) - add_custom_command( - OUTPUT ${STENO_DICT_SRC}.stamp - COMMAND ${Python3_EXECUTABLE} ${STENO_FETCH} - ${STENO_DICT_NAME} ${STENO_DICTS_DIR} - COMMAND ${CMAKE_COMMAND} -E touch ${STENO_DICT_SRC}.stamp - COMMENT "Checking ${STENO_DICT_NAME} dictionary for updates" + if(CONFIG_STENO_SPLIT_DICT) + if(CONFIG_ZMK_SPLIT_ROLE_CENTRAL) + # Central: behavior engine + local dict + BLE client + target_sources(app PRIVATE + src/behavior_steno.c + src/output.c + src/formatter.c + src/undo.c + src/dict_embed.S + src/dict_mphf.c + src/split_dict.c + src/split_cache.c + ) + else() + # Peripheral: local dict + GATT server for central queries + target_sources(app PRIVATE + src/dict_embed.S + src/dict_mphf.c + src/split_dict.c + src/split_cache.c ) - add_custom_target(steno_dict_fetch DEPENDS ${STENO_DICT_SRC}.stamp) endif() + elseif(NOT CONFIG_ZMK_SPLIT OR CONFIG_ZMK_SPLIT_ROLE_CENTRAL) + target_sources(app PRIVATE + src/behavior_steno.c + src/output.c + src/formatter.c + src/undo.c + src/dict_embed.S + src/dict_mphf.c + ) + endif() + + # Dict embed needed on: both halves in split mode, else central / + # non-split only. + set(STENO_NEED_DICT_EMBED FALSE) + if(CONFIG_STENO_SPLIT_DICT) + set(STENO_NEED_DICT_EMBED TRUE) + elseif(NOT CONFIG_ZMK_SPLIT OR CONFIG_ZMK_SPLIT_ROLE_CENTRAL) + set(STENO_NEED_DICT_EMBED TRUE) + endif() + + if(STENO_NEED_DICT_EMBED) + set(STENO_DICT_BIN ${CMAKE_CURRENT_BINARY_DIR}/steno_dict.bin) + steno_fetch_dict(${STENO_DICT_NAME} ${STENO_DICT_SRC}) if(CONFIG_STENO_SPLIT_DICT) - # Split mode: compile the appropriate partition if(CONFIG_ZMK_SPLIT_ROLE_CENTRAL) set(STENO_SPLIT_PART "left") set(STENO_SPLIT_SIZE ${CONFIG_STENO_DICT_LEFT_MAX_SIZE}) @@ -128,43 +199,39 @@ if(STENO_NEED_DICT_EMBED) --right-size ${CONFIG_STENO_DICT_RIGHT_MAX_SIZE} --max-size ${STENO_SPLIT_SIZE} --block-size 2048 - DEPENDS ${STENO_DICT_SRC} + DEPENDS ${STENO_DICT_SRC} ${STENO_DICT_SRC}.stamp ${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_mphf.py COMMENT "Compiling steno dictionary (${STENO_SPLIT_PART} partition)" ) - elseif(CONFIG_STENO_DICT_MPHF) + else() add_custom_command( OUTPUT ${STENO_DICT_BIN} COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_mphf.py ${STENO_DICT_SRC} ${STENO_DICT_BIN} --max-size ${CONFIG_STENO_DICT_MAX_SIZE} - DEPENDS ${STENO_DICT_SRC} + DEPENDS ${STENO_DICT_SRC} ${STENO_DICT_SRC}.stamp ${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_mphf.py COMMENT "Compiling steno dictionary (MPHF)" ) - else() - add_custom_command( - OUTPUT ${STENO_DICT_BIN} - COMMAND ${Python3_EXECUTABLE} - ${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_simple.py - ${STENO_DICT_SRC} -o ${STENO_DICT_BIN} - DEPENDS ${STENO_DICT_SRC} - ${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_simple.py - COMMENT "Compiling steno dictionary (simple)" - ) endif() add_custom_target(steno_dict_gen DEPENDS ${STENO_DICT_BIN}) - if(TARGET steno_dict_fetch) - add_dependencies(steno_dict_gen steno_dict_fetch) - endif() add_dependencies(app steno_dict_gen) - - file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/steno_dict_path.h - "#define STENO_DICT_BIN_PATH \"${STENO_DICT_BIN}\"\n") - target_include_directories(app PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) endif() + +endif() # CONFIG_STENO_DICT_V4 + +# ── Dict embed plumbing ────────────────────────────── +# dict_embed.S .incbin's the blob named by the generated header; +# OBJECT_DEPENDS makes it re-assemble when the blob changes. +if(STENO_DICT_BIN) + file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/steno_dict_path.h + "#define STENO_DICT_BIN_PATH \"${STENO_DICT_BIN}\"\n") + target_include_directories(app PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) + set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/src/dict_embed.S + TARGET_DIRECTORY app + PROPERTIES OBJECT_DEPENDS ${STENO_DICT_BIN}) endif() endif() # CONFIG_STENO_ENGINE diff --git a/Kconfig b/Kconfig index 5dd3b87..177fdb9 100644 --- a/Kconfig +++ b/Kconfig @@ -11,29 +11,43 @@ if STENO_ENGINE choice STENO_DICT prompt "Steno dictionary" - default STENO_DICT_PLOVER + default STENO_DICT_BOTH + +config STENO_DICT_BOTH + bool "Plover + Lapwing" + help + Compile both full dictionaries into the v4 union format. + The active dictionary is selected at runtime. config STENO_DICT_PLOVER - bool "Plover main dictionary (MPHF compressed)" + bool "Plover main dictionary" help - Use Plover main.json via MPHF compression. + Compile only the Plover main dictionary. config STENO_DICT_LAPWING - bool "Lapwing dictionary (MPHF compressed)" - -config STENO_DICT_TEST - bool "Test dictionary (46 entries, simple format)" + bool "Lapwing dictionary" help - Small built-in test dictionary for development. + Compile only the Lapwing dictionary. endchoice -config STENO_DICT_MPHF - bool - default y if STENO_DICT_PLOVER || STENO_DICT_LAPWING +config STENO_DICT_V4 + bool "v4 union split-section dictionary format" + default y select ZLIB help - Use MPHF (minimal perfect hash) dictionary format. + Compile the dictionary into the v4 union split-section format + (docs/FORMAT_V4.md): two half blobs, left (central) holds the + lookup structures, right (peripheral) holds the string table. + Requires STENO_SPLIT_DICT. Disable to fall back to the legacy + single-dictionary MPHF format. + +config STENO_DICT_MPHF + bool + default y if !STENO_DICT_V4 + select ZLIB + help + Legacy MPHF (minimal perfect hash) dictionary format. Selects ZLIB for block-compressed string table decompression. config STENO_CUSTOM_KEYMAP @@ -75,39 +89,43 @@ config STENO_MULTI_STROKE_TIMEOUT_MS config STENO_DICT_MAX_SIZE int "Max dictionary binary size (bytes)" + depends on !STENO_DICT_V4 default 473088 help - Max compiled dict size. 473088 = 462KB. - Build fails if dict exceeds this — no trimming. + Max compiled dict size for legacy MPHF single-blob builds. + 473088 = 462KB. Build FAILS if the dict exceeds this — + entries are never trimmed. menuconfig STENO_SPLIT_DICT bool "Split dictionary across both halves" default n - select STENO_DICT_MPHF help - Partition the steno dictionary across both keyboard halves. - Left (central) gets highest-importance entries for zero-latency - local lookup. Right (peripheral) gets remaining entries, queried - over BLE on local miss. Both halves have their own MPHF dict. + Store the steno dictionary across both keyboard halves. + v4 format: left (central) holds the lookup structures + (displacements, membership, fingerprints, conflicts, part of + the value index); right (peripheral) holds the string table, + queried over BLE. Legacy MPHF format: entries partitioned by + importance, right half queried over BLE on local miss. + Build FAILS if either half exceeds its budget — entries are + never trimmed. if STENO_SPLIT_DICT config STENO_DICT_LEFT_MAX_SIZE int "Left half dict budget (bytes)" - default 153600 + default 440320 help - Flash budget for left (central) partition. Default 150KB. - Central has more code overhead (behavior, formatter, output, - undo, BLE client) so gets a smaller dict partition. - Highest-importance entries fill this first. + Flash budget for the left (central) half blob. Default + 440320 = 430KB. Build FAILS if the compiled blob exceeds + this — entries are never trimmed. config STENO_DICT_RIGHT_MAX_SIZE int "Right half dict budget (bytes)" - default 512000 + default 527360 help - Flash budget for right (peripheral) partition. Default 500KB. - Peripheral has less code overhead so gets the bulk of the dict. - Remaining entries after left partition is filled. + Flash budget for the right (peripheral) half blob. Default + 527360 = 515KB. Build FAILS if the compiled blob exceeds + this — entries are never trimmed. config STENO_SPLIT_CACHE_SIZE int "LRU cache entries on central side" diff --git a/src/behavior_steno.c b/src/behavior_steno.c index 25599ee..863c63d 100644 --- a/src/behavior_steno.c +++ b/src/behavior_steno.c @@ -12,85 +12,115 @@ #include #include +#include + #include "output.h" #include "undo.h" #include "formatter.h" -#if IS_ENABLED(CONFIG_STENO_SPLIT_DICT) -#include "split_dict.h" -#include "dict_mphf.h" -#elif IS_ENABLED(CONFIG_STENO_DICT_MPHF) -#include "dict_mphf.h" -#else -#include "trie.h" -#endif - LOG_MODULE_DECLARE(zmk, CONFIG_ZMK_LOG_LEVEL); -extern const uint8_t _steno_dict_start[]; -extern const uint8_t _steno_dict_end[]; +/* + * Dictionary dispatcher — implemented by the active dict backend + * (dict_v4 / split). Exact-match lookup of `count` strokes: returns + * the translation length (>= 0, `out` NUL-terminated) on a hit, or a + * negative value on miss/error. steno_dict_max_strokes() reports the + * longest entry in the loaded dictionary (0 if none loaded). + */ +extern int steno_dict_lookup(const uint32_t *strokes, uint8_t count, + char *out, size_t out_size); +extern uint8_t steno_dict_max_strokes(void); -#if IS_ENABLED(CONFIG_STENO_DICT_MPHF) || IS_ENABLED(CONFIG_STENO_SPLIT_DICT) -static struct dict_mphf mphf_dict; -#endif +/* + * Runtime dictionary toggle (0 = plover, 1 = lapwing), implemented by + * the dict backend. + * + * TODO: wire steno_dict_set_active() to a dedicated chord or behavior + * parameter once a binding is assigned; no new DTS binding yet. + */ +extern void steno_dict_set_active(uint8_t dict_id); -#define STENO_MAX_MULTI 8 -#define STENO_MULTI_TIMEOUT_MS CONFIG_STENO_MULTI_STROKE_TIMEOUT_MS +#define STENO_BIT_STAR 9 struct steno_state { uint32_t current_chord; uint8_t keys_held; - uint32_t pending_strokes[STENO_MAX_MULTI]; - uint8_t stroke_count; - struct k_work_delayable multi_timeout; }; static struct steno_state state; static struct steno_fmt_state fmt_state; static struct stroke_history undo_history; -static bool dict_ready; -static void flush_strokes(void); -static void multi_timeout_handler(struct k_work *work); +/* ── formatter state snapshots (packed into history fmt_flags) ─── */ -static const char *do_lookup(const uint32_t *strokes, uint8_t count) +static uint8_t fmt_pack(const struct steno_fmt_state *s) { -#if IS_ENABLED(CONFIG_STENO_SPLIT_DICT) - /* Try local partition first (zero latency) */ - const char *local = dict_mphf_lookup(&mphf_dict, strokes, count); - if (local) { - return local; - } - /* Miss → query remote partition over BLE */ - static char split_buf[128]; - int ret = split_dict_lookup(strokes, count, split_buf, sizeof(split_buf)); - return (ret > 0) ? split_buf : NULL; -#elif IS_ENABLED(CONFIG_STENO_DICT_MPHF) - return dict_mphf_lookup(&mphf_dict, strokes, count); -#else - return steno_trie_lookup(strokes, count); -#endif + return (uint8_t)((s->space_pending ? 0x01 : 0) | + (s->cap_next ? 0x02 : 0) | + (s->upper_next ? 0x04 : 0) | + (s->lower_next ? 0x08 : 0) | + (s->suppress_space ? 0x10 : 0) | + (s->glue ? 0x20 : 0) | + (((uint8_t)s->mode & 0x03) << 6)); } -static bool do_has_prefix(const uint32_t *strokes, uint8_t count) +static void fmt_unpack(uint8_t flags, struct steno_fmt_state *s) { -#if IS_ENABLED(CONFIG_STENO_SPLIT_DICT) - /* Check local prefix table first */ - if (count == 1 && dict_mphf_has_prefix(&mphf_dict, strokes[0])) { - return true; - } - /* Fall through to remote */ - return split_dict_has_prefix(strokes, count); -#elif IS_ENABLED(CONFIG_STENO_DICT_MPHF) - return (count == 1) ? dict_mphf_has_prefix(&mphf_dict, strokes[0]) : false; -#else - return steno_trie_has_prefix(strokes, count); -#endif + s->space_pending = (flags & 0x01) != 0; + s->cap_next = (flags & 0x02) != 0; + s->upper_next = (flags & 0x04) != 0; + s->lower_next = (flags & 0x08) != 0; + s->suppress_space = (flags & 0x10) != 0; + s->glue = (flags & 0x20) != 0; + s->mode = (enum steno_fmt_mode)((flags >> 6) & 0x03); } +/* ── raw steno rendering (total-miss fallback) ──────────────────── */ + +/* '#' + 22 keys + '-' + NUL */ +#define STENO_RAW_MAX 26 + +/* Bit layout per parse_stroke (tools/compile_mphf.py, FORMAT_V4.md) */ +static const char steno_key_char[22] = { + 'S', 'T', 'K', 'P', 'W', 'H', 'R', /* bits 0-6: left */ + 'A', 'O', '*', 'E', 'U', /* bits 7-11: mid */ + 'F', 'R', 'P', 'B', 'L', 'G', 'T', 'S', 'D', 'Z', /* bits 12-21: right */ +}; + +#define STENO_MASK_MIDDLE 0x00000F80U /* A O * E U — implicit hyphen */ +#define STENO_MASK_NUM 0x00400000U /* # */ +#define STENO_FIRST_RIGHT 12 + +static void stroke_to_steno(uint32_t stroke, char out[STENO_RAW_MAX]) +{ + uint8_t len = 0; + bool implicit = (stroke & STENO_MASK_MIDDLE) != 0; + + if (stroke & STENO_MASK_NUM) { + out[len++] = '#'; + } + for (uint8_t bit = 0; bit < 22; bit++) { + if (!(stroke & (1U << bit))) { + continue; + } + if (bit >= STENO_FIRST_RIGHT && !implicit) { + out[len++] = '-'; + implicit = true; + } + out[len++] = steno_key_char[bit]; + } + out[len] = '\0'; +} + +/* ── emission ───────────────────────────────────────────────────── */ + static void emit_formatted(const char *translation, const uint32_t *strokes, uint8_t stroke_count) { + /* Snapshot formatter state BEFORE this translation mutates it, so + * undo/retrace can restore it. */ + uint8_t snap = fmt_pack(&fmt_state); + struct steno_fmt_result result; steno_fmt_process(&fmt_state, translation, &result); if (result.backspaces > 0) { @@ -100,98 +130,180 @@ static void emit_formatted(const char *translation, steno_output_send(result.text, result.len); } if (!result.is_command_only) { + uint16_t out_chars = (uint16_t)result.len + result.backspaces; + if (out_chars > 255) { + out_chars = 255; + } steno_undo_push(&undo_history, strokes, stroke_count, - result.len + result.backspaces, 0, 0); + (uint8_t)out_chars, 0, snap); } } +static void emit_raw_stroke(uint32_t stroke) +{ + char raw[STENO_RAW_MAX]; + stroke_to_steno(stroke, raw); + emit_formatted(raw, &stroke, 1); +} + +/* + * Greedy longest-match over a contiguous stroke span. Used to + * re-translate strokes orphaned by a retrace. No further retrace: + * matches are confined to the span. + */ +static void translate_span(const uint32_t *strokes, uint8_t count, + uint8_t max_win) +{ + uint8_t pos = 0; + + while (pos < count) { + uint8_t span = count - pos; + char text[STENO_FMT_MAX_OUTPUT]; + uint8_t match_len = 0; + + if (span > max_win) { + span = max_win; + } + for (uint8_t l = span; l >= 1; l--) { + if (steno_dict_lookup(&strokes[pos], l, text, sizeof(text)) >= 0) { + match_len = l; + break; + } + } + if (match_len == 0) { + emit_raw_stroke(strokes[pos]); + pos++; + } else { + emit_formatted(text, &strokes[pos], match_len); + pos += match_len; + } + } +} + +/* ── sliding longest-match translation with retrace ─────────────── */ + +static void translate_stroke(uint32_t stroke) +{ + uint8_t max_win = steno_dict_max_strokes(); + + if (max_win < 1) { + max_win = 1; + } else if (max_win > STENO_MAX_MULTI_STROKE) { + max_win = STENO_MAX_MULTI_STROKE; + } + + /* + * Build the lookup window: win[max_win - 1] is the new stroke, + * earlier strokes (pulled from translation history, newest first) + * fill leftward. avail = previous strokes actually gathered. + */ + uint32_t win[STENO_MAX_MULTI_STROKE]; + uint8_t avail = 0; + + win[max_win - 1] = stroke; + for (uint16_t i = 0; avail < max_win - 1; i++) { + const struct stroke_history_entry *e = + steno_undo_peek_at(&undo_history, i); + if (!e) { + break; + } + for (uint8_t k = e->stroke_count; k > 0 && avail < max_win - 1; k--) { + avail++; + win[max_win - 1 - avail] = e->strokes[k - 1]; + } + } + + /* Longest match wins: L = avail + 1 down to 1 */ + char text[STENO_FMT_MAX_OUTPUT]; + uint8_t match_len = 0; + + for (uint8_t l = avail + 1; l >= 1; l--) { + if (steno_dict_lookup(&win[max_win - l], l, text, sizeof(text)) >= 0) { + match_len = l; + break; + } + } + + if (match_len == 0) { + /* Total miss → raw steno chars for this stroke */ + emit_raw_stroke(stroke); + return; + } + + if (match_len == 1) { + emit_formatted(text, &stroke, 1); + return; + } + + /* + * Retrace: the match swallows strokes already emitted by earlier + * translations. Pop those, erase their output, restore the + * formatter state that preceded the oldest popped entry, then + * re-emit. + */ + uint8_t needed = match_len - 1; + uint8_t covered = 0; + uint16_t erase = 0; + uint8_t snap = fmt_pack(&fmt_state); + uint32_t orphan[STENO_MAX_MULTI_STROKE]; + uint8_t orphan_count = 0; + + while (covered < needed) { + struct stroke_history_entry *e = steno_undo_pop(&undo_history); + if (!e) { + /* Cannot happen: avail came from the same entries */ + break; + } + covered += e->stroke_count; + erase += e->output_len; + snap = e->fmt_flags; + if (covered > needed) { + /* + * Oldest popped entry straddles the window boundary: its + * leading strokes fall outside the new match and must be + * re-translated. Copy before pushes reuse the ring slot. + */ + orphan_count = covered - needed; + memcpy(orphan, e->strokes, orphan_count * sizeof(uint32_t)); + } + } + + if (erase > 0) { + steno_output_backspace(erase); + } + fmt_unpack(snap, &fmt_state); + + if (orphan_count > 0) { + translate_span(orphan, orphan_count, max_win); + } + + emit_formatted(text, &win[max_win - match_len], match_len); +} + +/* ── chord assembly ─────────────────────────────────────────────── */ + static void process_chord(void) { - if (state.current_chord == 0) { + uint32_t stroke = state.current_chord; + + state.current_chord = 0; + if (stroke == 0) { return; } - /* Star-only stroke (bit 9) → undo */ - if (state.current_chord == (1U << 9)) { - state.current_chord = 0; + /* Star-only stroke → undo last translation */ + if (stroke == (1U << STENO_BIT_STAR)) { struct stroke_history_entry *ue = steno_undo_pop(&undo_history); if (ue) { - steno_output_backspace(ue->output_len); + if (ue->output_len > 0) { + steno_output_backspace(ue->output_len); + } + fmt_unpack(ue->fmt_flags, &fmt_state); } return; } - k_work_cancel_delayable(&state.multi_timeout); - - if (state.stroke_count < STENO_MAX_MULTI) { - state.pending_strokes[state.stroke_count] = state.current_chord; - state.stroke_count++; - } else { - flush_strokes(); - state.pending_strokes[0] = state.current_chord; - state.stroke_count = 1; - } - state.current_chord = 0; - - if (!dict_ready) { - flush_strokes(); - return; - } - - const char *translation = do_lookup(state.pending_strokes, state.stroke_count); - - if (translation) { - emit_formatted(translation, state.pending_strokes, state.stroke_count); - state.stroke_count = 0; - return; - } - - if (do_has_prefix(state.pending_strokes, state.stroke_count)) { - k_work_schedule(&state.multi_timeout, - K_MSEC(STENO_MULTI_TIMEOUT_MS)); - return; - } - - if (state.stroke_count > 1) { - uint32_t last = state.pending_strokes[state.stroke_count - 1]; - state.stroke_count--; - - const char *partial = do_lookup(state.pending_strokes, state.stroke_count); - if (partial) { - emit_formatted(partial, state.pending_strokes, state.stroke_count); - } - - state.pending_strokes[0] = last; - state.stroke_count = 1; - - const char *rest = do_lookup(&last, 1); - if (rest) { - emit_formatted(rest, &last, 1); - state.stroke_count = 0; - } - return; - } - - flush_strokes(); -} - -static void flush_strokes(void) -{ - LOG_DBG("Flushing %u untranslated strokes", state.stroke_count); - state.stroke_count = 0; -} - -static void multi_timeout_handler(struct k_work *work) -{ - ARG_UNUSED(work); - if (state.stroke_count == 0) { - return; - } - const char *translation = do_lookup(state.pending_strokes, state.stroke_count); - if (translation) { - emit_formatted(translation, state.pending_strokes, state.stroke_count); - } - state.stroke_count = 0; + translate_stroke(stroke); } static int on_steno_binding_pressed(struct zmk_behavior_binding *binding, @@ -228,49 +340,9 @@ static int behavior_steno_init(const struct device *dev) state.current_chord = 0; state.keys_held = 0; - state.stroke_count = 0; steno_fmt_init(&fmt_state); steno_undo_init(&undo_history); - k_work_init_delayable(&state.multi_timeout, multi_timeout_handler); - -#if IS_ENABLED(CONFIG_STENO_SPLIT_DICT) - /* Init local partition dict */ - { - size_t dict_size = _steno_dict_end - _steno_dict_start; - if (dict_size > 4) { - int ret = dict_mphf_init(&mphf_dict, _steno_dict_start, dict_size); - if (ret == 0) { - LOG_INF("Local partition loaded (%u bytes)", (unsigned)dict_size); - } else { - LOG_ERR("Local partition init failed: %d", ret); - } - } else { - LOG_WRN("No local partition embedded"); - } - } - /* Init BLE client for remote partition */ - split_dict_init(); - dict_ready = true; -#else - size_t dict_size = _steno_dict_end - _steno_dict_start; - if (dict_size > 4) { - int ret; -#if IS_ENABLED(CONFIG_STENO_DICT_MPHF) - ret = dict_mphf_init(&mphf_dict, _steno_dict_start, dict_size); -#else - ret = steno_trie_init(_steno_dict_start, dict_size); -#endif - if (ret == 0) { - dict_ready = true; - LOG_INF("Steno dict loaded (%u bytes)", (unsigned)dict_size); - } else { - LOG_ERR("Steno dict init failed: %d", ret); - } - } else { - LOG_WRN("No steno dict embedded"); - } -#endif LOG_INF("Steno engine initialized"); return 0; diff --git a/src/dict_embed.S b/src/dict_embed.S index 578178d..a5734b3 100644 --- a/src/dict_embed.S +++ b/src/dict_embed.S @@ -1,14 +1,30 @@ +/* + * Copyright (c) 2024 Afiq Zudin Hadi + * SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + * + * Embeds the compiled dictionary blob selected by the build system. + * steno_dict_path.h is generated by CMake and names the blob for this + * half (v4: steno_v4_left.bin / steno_v4_right.bin; legacy MPHF: + * steno_dict.bin). Both symbol spellings are exported: legacy code + * uses _steno_dict_start/_steno_dict_end, dict_v4.c uses + * steno_dict_start/steno_dict_end. + */ + #include "steno_dict_path.h" .section .rodata.steno_dict, "a", %progbits .global _steno_dict_start .global _steno_dict_end + .global steno_dict_start + .global steno_dict_end .balign 4 _steno_dict_start: +steno_dict_start: #ifdef STENO_DICT_BIN_PATH .incbin STENO_DICT_BIN_PATH #else .byte 0x00, 0x00, 0x00, 0x00 #endif _steno_dict_end: +steno_dict_end: diff --git a/src/dict_v4.c b/src/dict_v4.c new file mode 100644 index 0000000..9286750 --- /dev/null +++ b/src/dict_v4.c @@ -0,0 +1,839 @@ +/* + * Copyright (c) 2024 zmk-steno-engine contributors + * SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + * + * Licensed under the PolyForm Noncommercial License 1.0.0; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://polyformproject.org/licenses/noncommercial/1.0.0 + */ + +/* + * Dictionary binary format v4 decoder — implementation. + * + * All data is read directly from flash. No heap allocation on the + * Zephyr path; string blocks are inflated on demand into a static + * two-slot LRU cache (16 KB per slot, the compile-time block bound). + */ + +#include "dict_v4.h" +#include +#include +#include + +#define DICT_V4_ARRAY_LEN(a) (sizeof(a) / sizeof((a)[0])) + +/* ─── Raw-deflate block inflate (wbits = -15) ─── */ + +#ifndef __ZEPHYR__ + +/* Use zlib on host for native tests */ +#ifdef HAS_ZLIB +#include +static int block_inflate_raw(const uint8_t *src, size_t src_len, + uint8_t *dst, size_t dst_cap, size_t *dst_len) +{ + z_stream strm; + memset(&strm, 0, sizeof(strm)); + if (inflateInit2(&strm, -15) != Z_OK) { + return -1; + } + strm.next_in = (Bytef *)(uintptr_t)src; + strm.avail_in = (uInt)src_len; + strm.next_out = dst; + strm.avail_out = (uInt)dst_cap; + int ret = inflate(&strm, Z_FINISH); + inflateEnd(&strm); + if (ret == Z_STREAM_END) { + *dst_len = dst_cap - strm.avail_out; + return 0; + } + return -1; +} +#else +static int block_inflate_raw(const uint8_t *src, size_t src_len, + uint8_t *dst, size_t dst_cap, size_t *dst_len) +{ + /* No zlib on host — string sections unavailable */ + (void)src; (void)src_len; (void)dst; (void)dst_cap; (void)dst_len; + return -1; +} +#endif /* HAS_ZLIB */ + +#else /* __ZEPHYR__ */ + +#include + +#if __has_include() +#include +#elif __has_include() +#include +#endif + +static int block_inflate_raw(const uint8_t *src, size_t src_len, + uint8_t *dst, size_t dst_cap, size_t *dst_len) +{ +#if defined(CONFIG_ZLIB) + z_stream strm; + memset(&strm, 0, sizeof(strm)); + if (inflateInit2(&strm, -15) != Z_OK) { + return -1; + } + strm.next_in = (Bytef *)(uintptr_t)src; + strm.avail_in = (uInt)src_len; + strm.next_out = dst; + strm.avail_out = (uInt)dst_cap; + int ret = inflate(&strm, Z_FINISH); + inflateEnd(&strm); + if (ret == Z_STREAM_END) { + *dst_len = dst_cap - strm.avail_out; + return 0; + } + return -1; +#else + /* No zlib available — the v4 string path cannot run on this half */ + (void)src; (void)src_len; (void)dst; (void)dst_cap; (void)dst_len; + return -1; +#endif +} + +#endif /* __ZEPHYR__ */ + +/* ─── Byte helpers (blob fields may be unaligned) ─── */ + +static uint32_t get_le32(const uint8_t *p) +{ + return (uint32_t)p[0] + | ((uint32_t)p[1] << 8) + | ((uint32_t)p[2] << 16) + | ((uint32_t)p[3] << 24); +} + +/* ─── FNV-1a 32-bit hash (identical to v2) ─── */ + +static uint32_t fnv1a_32(const uint8_t *data, size_t len) +{ + uint32_t h = 0x811c9dc5u; + for (size_t i = 0; i < len; i++) { + h ^= data[i]; + h *= 0x01000193u; + } + return h; +} + +static uint32_t hash_key(const uint8_t *key, size_t key_len, uint32_t seed) +{ + uint32_t h = 0x811c9dc5u; + uint8_t seed_bytes[4]; + seed_bytes[0] = (uint8_t)(seed); + seed_bytes[1] = (uint8_t)(seed >> 8); + seed_bytes[2] = (uint8_t)(seed >> 16); + seed_bytes[3] = (uint8_t)(seed >> 24); + for (int i = 0; i < 4; i++) { + h ^= seed_bytes[i]; + h *= 0x01000193u; + } + for (size_t i = 0; i < key_len; i++) { + h ^= key[i]; + h *= 0x01000193u; + } + return h; +} + +/* ─── LSB-first bit-packed field reading (MEMBERSHIP/VALIDX/FP) ─── */ + +static uint32_t read_bits_lsb(const uint8_t *data, uint32_t bit_pos, uint8_t n_bits) +{ + uint32_t value = 0; + for (uint8_t i = 0; i < n_bits; i++) { + uint32_t byte_idx = (bit_pos + i) / 8; + uint8_t bit_idx = (bit_pos + i) % 8; + if (data[byte_idx] & (1u << bit_idx)) { + value |= (1u << i); + } + } + return value; +} + +/* ─── MSB-first bit reading (DISP Huffman stream only) ─── */ + +static inline uint32_t read_bit_msb(const uint8_t *data, uint32_t bit_pos) +{ + return (data[bit_pos >> 3] >> (7 - (bit_pos & 7))) & 1u; +} + +/* ─── Canonical Huffman (DISP displacement classes) ─── */ + +static int huff_build(struct dict_v4 *d) +{ + const uint8_t *code_len = d->disp_code_len; + uint8_t max_len = 0; + + memset(d->huff_count, 0, sizeof(d->huff_count)); + + for (int c = 0; c < DICT_V4_DISP_CLASSES; c++) { + uint8_t l = code_len[c]; + if (l == 0) { + continue; + } + if (l > DICT_V4_DISP_CLASSES) { + return -EBADMSG; + } + d->huff_count[l]++; + if (l > max_len) { + max_len = l; + } + } + if (max_len == 0) { + return -EBADMSG; + } + + /* Symbol list in canonical (code_len, class) order */ + uint8_t idx = 0; + for (uint8_t l = 1; l <= max_len; l++) { + d->huff_sym_base[l] = idx; + for (uint8_t c = 0; c < DICT_V4_DISP_CLASSES; c++) { + if (code_len[c] == l) { + d->huff_symbols[idx++] = c; + } + } + } + + /* First canonical code per length; codes assigned numerically + * increasing, MSB-first prefix codes. */ + uint32_t code = 0; + for (uint8_t l = 1; l <= max_len; l++) { + d->huff_first_code[l] = code; + code += d->huff_count[l]; + if (code > (1u << l)) { + return -EBADMSG; /* over-subscribed code space */ + } + code <<= 1; + } + + d->huff_max_len = max_len; + return 0; +} + +/* Decode one displacement value at *bit_pos, advancing it. */ +static int huff_decode_one(const struct dict_v4 *d, uint32_t *bit_pos, uint32_t *v_out) +{ + uint32_t pos = *bit_pos; + uint32_t code = 0; + uint8_t sym = 0; + uint8_t found = 0; + + for (uint8_t len = 1; len <= d->huff_max_len; len++) { + if (pos >= d->disp_stream_bits) { + return -EBADMSG; + } + code = (code << 1) | read_bit_msb(d->disp_stream, pos++); + if (d->huff_count[len] != 0 && + code >= d->huff_first_code[len] && + code - d->huff_first_code[len] < d->huff_count[len]) { + sym = d->huff_symbols[d->huff_sym_base[len] + + (code - d->huff_first_code[len])]; + found = 1; + break; + } + } + if (!found) { + return -EBADMSG; + } + + uint32_t v; + if (sym <= 1) { + v = sym; + } else { + uint32_t extra = 0; + for (uint8_t i = 0; i < sym - 1; i++) { + if (pos >= d->disp_stream_bits) { + return -EBADMSG; + } + extra = (extra << 1) | read_bit_msb(d->disp_stream, pos++); + } + v = (1u << (sym - 1)) | extra; + } + + *bit_pos = pos; + *v_out = v; + return 0; +} + +/* Random access: start at skip[b >> 8], decode (b & 255) values, next + * value is bucket b's displacement. */ +static int disp_decode(const struct dict_v4 *d, uint32_t bucket, uint32_t *v_out) +{ + uint32_t skip_idx = bucket >> 8; + + if (skip_idx >= d->disp_skip_count) { + return -EBADMSG; + } + + uint32_t bit_pos = get_le32(d->disp_skip + skip_idx * 4); + uint32_t v = 0; + + for (uint32_t i = 0; i <= (bucket & 255u); i++) { + int ret = huff_decode_one(d, &bit_pos, &v); + if (ret != 0) { + return ret; + } + } + + *v_out = v; + return 0; +} + +/* ─── CONFLICTS: binary search on sorted 5-byte records ─── */ + +static int conflict_lookup(const struct dict_v4 *d, uint32_t slot, uint32_t *string_id) +{ + uint32_t lo = 0; + uint32_t hi = d->conflict_count; + + while (lo < hi) { + uint32_t mid = lo + (hi - lo) / 2; + const uint8_t *rec = d->conflicts + mid * DICT_V4_CONFLICT_REC_SIZE; + uint64_t v = (uint64_t)rec[0] + | ((uint64_t)rec[1] << 8) + | ((uint64_t)rec[2] << 16) + | ((uint64_t)rec[3] << 24) + | ((uint64_t)rec[4] << 32); + uint32_t rec_slot = (uint32_t)(v & 0x3FFFFu); + + if (rec_slot == slot) { + *string_id = (uint32_t)(v >> 18); + return 0; + } + if (rec_slot < slot) { + lo = mid + 1; + } else { + hi = mid; + } + } + return -ENOENT; +} + +/* ─── Init ─── */ + +static int init_disp(struct dict_v4 *d, const uint8_t *sec, uint32_t sec_len) +{ + uint32_t fixed = DICT_V4_DISP_CLASSES + 4; + + if (sec_len < fixed) { + return -EBADMSG; + } + + uint32_t skip_count = get_le32(sec + DICT_V4_DISP_CLASSES); + uint32_t expected = (d->header->bucket_count + 255u) / 256u; + + if (skip_count != expected || sec_len < fixed + skip_count * 4) { + return -EBADMSG; + } + + d->disp_code_len = sec; + d->disp_skip = sec + fixed; + d->disp_skip_count = skip_count; + d->disp_stream = sec + fixed + skip_count * 4; + d->disp_stream_bits = (sec_len - fixed - skip_count * 4) * 8; + + return huff_build(d); +} + +int dict_v4_init(struct dict_v4 *d, const void *blob, size_t len) +{ + if (!d || !blob) { + return -EINVAL; + } + + memset(d, 0, sizeof(*d)); + + if (len < sizeof(struct dict_v4_header)) { + return -EBADMSG; + } + + const struct dict_v4_header *hdr = (const struct dict_v4_header *)blob; + if (hdr->magic != DICT_V4_MAGIC || hdr->version != DICT_V4_VERSION) { + return -EBADMSG; + } + + const uint8_t *base = (const uint8_t *)blob; + uint32_t sc = hdr->section_count; + uint32_t dir_end = sizeof(struct dict_v4_header) + + sc * sizeof(struct dict_v4_section); + + if (len < dir_end) { + return -EBADMSG; + } + + d->header = hdr; + + for (uint32_t i = 0; i < sc; i++) { + const struct dict_v4_section *ent = (const struct dict_v4_section *) + (base + sizeof(struct dict_v4_header) + i * sizeof(*ent)); + const uint8_t *sec = base + ent->offset; + uint32_t sec_len = ent->len; + int ret; + + if (ent->offset < dir_end || (ent->offset & 3u) != 0 || + ent->offset > len || sec_len > len - ent->offset) { + return -EBADMSG; + } + + switch (ent->type) { + case DICT_V4_SEC_DISP: + ret = init_disp(d, sec, sec_len); + if (ret != 0) { + return ret; + } + break; + + case DICT_V4_SEC_MEMBERSHIP: + if ((uint64_t)sec_len * 8 < (uint64_t)hdr->n * 2) { + return -EBADMSG; + } + d->membership = sec; + break; + + case DICT_V4_SEC_VALIDX: { + uint32_t count = (uint32_t)(((uint64_t)sec_len * 8) / + DICT_V4_VALIDX_BITS); + if (ent->param > hdr->n || count > hdr->n - ent->param) { + return -EBADMSG; + } + d->validx = sec; + d->validx_start_slot = ent->param; + d->validx_slot_count = count; + break; + } + + case DICT_V4_SEC_CONFLICTS: + if (ent->param != hdr->conflict_count || + sec_len < ent->param * DICT_V4_CONFLICT_REC_SIZE) { + return -EBADMSG; + } + d->conflicts = sec; + d->conflict_count = ent->param; + break; + + case DICT_V4_SEC_FP: + if ((uint64_t)sec_len * 8 < (uint64_t)hdr->n * hdr->fp_bits) { + return -EBADMSG; + } + d->fp = sec; + break; + + case DICT_V4_SEC_STRDIR: { + if (sec_len < 4) { + return -EBADMSG; + } + uint32_t block_count = get_le32(sec); + if (block_count != ent->param || + sec_len < 4 + (uint64_t)block_count * 8) { + return -EBADMSG; + } + d->strdir_entries = sec + 4; + d->strdir_block_count = block_count; + break; + } + + case DICT_V4_SEC_STRINGS: + d->strings = sec; + d->strings_len = sec_len; + break; + + default: + /* Unknown section: ignore for forward compatibility */ + break; + } + } + + return 0; +} + +/* ─── Lookup (decision path, left half) ─── */ + +int dict_v4_lookup(const struct dict_v4 *d, const uint32_t *strokes, uint8_t count, + uint8_t active_dict, uint32_t *slot, uint32_t *string_id) +{ + if (!d || !d->header || !strokes || count == 0 || !slot || active_dict > 1) { + return -EINVAL; + } + if (!d->disp_code_len || !d->membership || !d->fp || !d->validx) { + return -ENOTSUP; /* this half lacks the decision sections */ + } + + const struct dict_v4_header *hdr = d->header; + + if (!(hdr->dicts_mask & (1u << active_dict))) { + return DICT_V4_MISS; + } + if (count > hdr->max_entry_strokes || count > DICT_V4_MAX_KEY_STROKES) { + return DICT_V4_MISS; + } + + uint8_t key_buf[DICT_V4_MAX_KEY_STROKES * 4]; + size_t key_len = (size_t)count * 4; + + for (uint8_t i = 0; i < count; i++) { + key_buf[i * 4 + 0] = (uint8_t)(strokes[i]); + key_buf[i * 4 + 1] = (uint8_t)(strokes[i] >> 8); + key_buf[i * 4 + 2] = (uint8_t)(strokes[i] >> 16); + key_buf[i * 4 + 3] = (uint8_t)(strokes[i] >> 24); + } + + uint32_t bucket = hash_key(key_buf, key_len, 0) % hdr->bucket_count; + uint32_t v; + int ret = disp_decode(d, bucket, &v); + if (ret != 0) { + return ret; + } + + uint32_t s; + if (v >= hdr->d_threshold) { + s = v - hdr->d_threshold; /* direct slot index */ + if (s >= hdr->n) { + return -EBADMSG; + } + } else { + s = hash_key(key_buf, key_len, v + 1) % hdr->n; + } + + uint8_t expected_fp = (uint8_t)(fnv1a_32(key_buf, key_len) & + ((1u << hdr->fp_bits) - 1u)); + if (read_bits_lsb(d->fp, s * hdr->fp_bits, hdr->fp_bits) != expected_fp) { + return DICT_V4_MISS; + } + + uint32_t m = read_bits_lsb(d->membership, s * 2, 2); + if (!(m & (1u << active_dict))) { + return DICT_V4_MISS; + } + + *slot = s; + + if (s < d->validx_start_slot || + s - d->validx_start_slot >= d->validx_slot_count) { + return DICT_V4_FOUND_REMOTE; + } + + uint32_t sid = read_bits_lsb(d->validx, + (s - d->validx_start_slot) * DICT_V4_VALIDX_BITS, + DICT_V4_VALIDX_BITS); + + /* Main value index holds plover's string id for both-membership + * conflicts; substitute lapwing's from the conflict table. */ + if (active_dict == DICT_V4_DICT_LAPWING && m == 3 && d->conflicts) { + uint32_t conflict_sid; + if (conflict_lookup(d, s, &conflict_sid) == 0) { + sid = conflict_sid; + } + } + + if (sid >= hdr->string_count) { + return -EBADMSG; + } + if (string_id) { + *string_id = sid; + } + return DICT_V4_FOUND_LOCAL; +} + +/* ─── String path (right half or host test) ─── */ + +/* Static two-slot LRU block cache. 16 KB per slot: the compiler caps + * each front-coded block's uncompressed size at 16384 bytes. */ +struct dict_v4_block_slot { + const struct dict_v4 *owner; + uint32_t block_idx; + uint32_t len; + uint32_t tick; + uint8_t buf[DICT_V4_BLOCK_BUF_SIZE]; +}; + +static struct dict_v4_block_slot block_cache[2]; +static uint32_t block_cache_tick; + +static uint32_t strdir_comp_off(const struct dict_v4 *d, uint32_t block) +{ + return get_le32(d->strdir_entries + block * 8); +} + +static uint32_t strdir_first_sid(const struct dict_v4 *d, uint32_t block) +{ + return get_le32(d->strdir_entries + block * 8 + 4); +} + +static int block_get(const struct dict_v4 *d, uint32_t block, + const uint8_t **buf, uint32_t *buf_len) +{ + struct dict_v4_block_slot *victim = &block_cache[0]; + + for (size_t i = 0; i < DICT_V4_ARRAY_LEN(block_cache); i++) { + struct dict_v4_block_slot *slot = &block_cache[i]; + + if (slot->owner == d && slot->block_idx == block) { + slot->tick = ++block_cache_tick; + *buf = slot->buf; + *buf_len = slot->len; + return 0; + } + if (slot->tick < victim->tick) { + victim = slot; + } + } + + uint32_t comp_off = strdir_comp_off(d, block); + uint32_t comp_end = (block + 1 < d->strdir_block_count) + ? strdir_comp_off(d, block + 1) + : d->strings_len; + + if (comp_end < comp_off || comp_end > d->strings_len) { + return -EBADMSG; + } + + size_t out_len; + victim->owner = NULL; + if (block_inflate_raw(d->strings + comp_off, comp_end - comp_off, + victim->buf, sizeof(victim->buf), &out_len) != 0) { + return -EIO; + } + + victim->owner = d; + victim->block_idx = block; + victim->len = (uint32_t)out_len; + victim->tick = ++block_cache_tick; + + *buf = victim->buf; + *buf_len = victim->len; + return 0; +} + +int dict_v4_string_by_id_ctx(const struct dict_v4 *d, uint32_t string_id, + char *out, size_t out_size) +{ + if (!d || !d->header || !out || out_size == 0) { + return -EINVAL; + } + if (!d->strdir_entries || !d->strings) { + return -ENOTSUP; /* this half lacks the string sections */ + } + if (string_id >= d->header->string_count || d->strdir_block_count == 0) { + return -ENOENT; + } + + /* Binary search: last block with first_string_id <= string_id */ + uint32_t lo = 0; + uint32_t hi = d->strdir_block_count; + while (lo < hi) { + uint32_t mid = lo + (hi - lo) / 2; + if (strdir_first_sid(d, mid) <= string_id) { + lo = mid + 1; + } else { + hi = mid; + } + } + if (lo == 0) { + return -EBADMSG; + } + uint32_t block = lo - 1; + + const uint8_t *buf; + uint32_t buf_len; + int ret = block_get(d, block, &buf, &buf_len); + if (ret != 0) { + return ret; + } + + /* Walk the front-coded chain, maintaining the previous string + * in place: the shared prefix bytes are already correct. */ + static uint8_t prev[DICT_V4_MAX_TEXT + 1]; + uint32_t steps = string_id - strdir_first_sid(d, block); + uint32_t pos = 0; + uint32_t prev_len = 0; + + for (uint32_t i = 0; i <= steps; i++) { + if (pos >= buf_len) { + return -EBADMSG; + } + uint32_t prefix = buf[pos++]; + if (prefix > prev_len || (i == 0 && prefix != 0)) { + return -EBADMSG; + } + uint32_t cur_len = prefix; + while (pos < buf_len && buf[pos] != 0) { + if (cur_len >= DICT_V4_MAX_TEXT) { + return -EBADMSG; + } + prev[cur_len++] = buf[pos++]; + } + if (pos >= buf_len) { + return -EBADMSG; /* missing terminator */ + } + pos++; /* skip 0x00 */ + prev_len = cur_len; + } + + if (prev_len + 1 > out_size) { + return -ENOSPC; + } + memcpy(out, prev, prev_len); + out[prev_len] = '\0'; + return (int)prev_len; +} + +int dict_v4_resolve_slot_ctx(const struct dict_v4 *d, uint32_t slot, uint8_t dict, + char *out, size_t out_size) +{ + if (!d || !d->header || !out || out_size == 0 || dict > 1) { + return -EINVAL; + } + if (!d->validx) { + return -ENOTSUP; + } + if (slot < d->validx_start_slot || + slot - d->validx_start_slot >= d->validx_slot_count) { + return -ENOENT; + } + + uint32_t sid = read_bits_lsb(d->validx, + (slot - d->validx_start_slot) * + DICT_V4_VALIDX_BITS, + DICT_V4_VALIDX_BITS); + + if (dict == DICT_V4_DICT_LAPWING && d->conflicts) { + uint32_t conflict_sid; + if (conflict_lookup(d, slot, &conflict_sid) == 0) { + sid = conflict_sid; + } + } + + if (sid >= d->header->string_count) { + return -EBADMSG; + } + return dict_v4_string_by_id_ctx(d, sid, out, out_size); +} + +uint8_t dict_v4_max_strokes(const struct dict_v4 *d) +{ + if (!d || !d->header) { + return 0; + } + return d->header->max_entry_strokes; +} + +/* ─── Zephyr singleton wrappers (embedded blob) ─── */ + +#ifdef __ZEPHYR__ + +#ifdef CONFIG_STENO_SPLIT_DICT +/* BLE transport for the sections living on the other half (split_dict.c) */ +extern int split_dict_get_string(uint32_t string_id, char *out, size_t out_size); +extern int split_dict_resolve(uint32_t slot, uint8_t dict, char *out, size_t out_size); +#endif + +extern const uint8_t steno_dict_start[]; +extern const uint8_t steno_dict_end[]; + +static struct dict_v4 dict_singleton; +static bool dict_singleton_ready; +static uint8_t active_dict_id; + +static int singleton_init(void) +{ + if (dict_singleton_ready) { + return 0; + } + + size_t blob_len = (size_t)(steno_dict_end - steno_dict_start); + int ret = dict_v4_init(&dict_singleton, steno_dict_start, blob_len); + if (ret != 0) { + return ret; + } + + active_dict_id = (dict_singleton.header->dicts_mask & + (1u << DICT_V4_DICT_PLOVER)) + ? DICT_V4_DICT_PLOVER : DICT_V4_DICT_LAPWING; + dict_singleton_ready = true; + return 0; +} + +int dict_v4_string_by_id(uint32_t string_id, char *out, size_t out_size) +{ + int ret = singleton_init(); + if (ret != 0) { + return ret; + } + return dict_v4_string_by_id_ctx(&dict_singleton, string_id, out, out_size); +} + +int dict_v4_resolve_slot(uint32_t slot, uint8_t dict, char *out, size_t out_size) +{ + int ret = singleton_init(); + if (ret != 0) { + return ret; + } + return dict_v4_resolve_slot_ctx(&dict_singleton, slot, dict, out, out_size); +} + +int steno_dict_lookup(const uint32_t *strokes, uint8_t count, + char *out, size_t out_size) +{ + int ret = singleton_init(); + if (ret != 0) { + return ret; + } + + uint32_t slot = 0; + uint32_t string_id = 0; + + ret = dict_v4_lookup(&dict_singleton, strokes, count, active_dict_id, + &slot, &string_id); + + if (ret == DICT_V4_FOUND_LOCAL) { + if (dict_singleton.strdir_entries && dict_singleton.strings) { + return dict_v4_string_by_id_ctx(&dict_singleton, string_id, + out, out_size); + } +#ifdef CONFIG_STENO_SPLIT_DICT + return split_dict_get_string(string_id, out, out_size); +#else + return -ENOTSUP; +#endif + } + + if (ret == DICT_V4_FOUND_REMOTE) { +#ifdef CONFIG_STENO_SPLIT_DICT + return split_dict_resolve(slot, active_dict_id, out, out_size); +#else + return -ENOTSUP; +#endif + } + + if (ret == DICT_V4_MISS) { + return -ENOENT; + } + return ret; +} + +uint8_t steno_dict_max_strokes(void) +{ + if (singleton_init() != 0) { + return 0; + } + return dict_v4_max_strokes(&dict_singleton); +} + +void steno_dict_set_active(uint8_t dict_id) +{ + if (singleton_init() != 0) { + return; + } + if (dict_id > 1 || + !(dict_singleton.header->dicts_mask & (1u << dict_id))) { + return; + } + active_dict_id = dict_id; +} + +#endif /* __ZEPHYR__ */ diff --git a/src/dict_v4.h b/src/dict_v4.h new file mode 100644 index 0000000..d23bca6 --- /dev/null +++ b/src/dict_v4.h @@ -0,0 +1,175 @@ +/* + * Copyright (c) 2024 zmk-steno-engine contributors + * SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + * + * Licensed under the PolyForm Noncommercial License 1.0.0; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://polyformproject.org/licenses/noncommercial/1.0.0 + */ + +/* + * Dictionary binary format v4 — "Union Split-Section" decoder. + * + * Byte layout per docs/FORMAT_V4.md. One union CHD MPHF over both + * source dicts (plover = 0, lapwing = 1), sections split across the + * two keyboard halves: + * + * LEFT: DISP, MEMBERSHIP, FP, CONFLICTS, VALIDX slice [0, k) + * RIGHT: STRDIR, STRINGS, CONFLICTS, VALIDX slice [k, n) + * + * The decision path (dict_v4_lookup) runs entirely on the left half; + * the string path (dict_v4_string_by_id_ctx / dict_v4_resolve_slot_ctx) + * runs on whichever context holds the needed sections. + */ + +#ifndef DICT_V4_H +#define DICT_V4_H + +#include +#include + +#define DICT_V4_MAGIC 0x344E5453u /* "STN4" */ +#define DICT_V4_VERSION 4 + +/* Section types (FORMAT_V4.md section 4) */ +#define DICT_V4_SEC_DISP 1 +#define DICT_V4_SEC_MEMBERSHIP 2 +#define DICT_V4_SEC_VALIDX 3 +#define DICT_V4_SEC_CONFLICTS 4 +#define DICT_V4_SEC_FP 5 +#define DICT_V4_SEC_STRDIR 6 +#define DICT_V4_SEC_STRINGS 7 + +/* Source dictionary ids */ +#define DICT_V4_DICT_PLOVER 0 +#define DICT_V4_DICT_LAPWING 1 + +/* Displacement Huffman classes: class = bit_length(v), v <= 2^18-ish */ +#define DICT_V4_DISP_CLASSES 19 + +/* VALIDX fixed field width */ +#define DICT_V4_VALIDX_BITS 17 + +/* CONFLICTS record: u40le = slot (18 bits) | string_id << 18 */ +#define DICT_V4_CONFLICT_REC_SIZE 5 + +/* Every translation is <= 255 bytes (compile-time enforced) */ +#define DICT_V4_MAX_TEXT 255 + +/* FC string block decode buffer bound (FORMAT_V4.md section 4.7) */ +#define DICT_V4_BLOCK_BUF_SIZE 16384 + +/* Local key buffer bound; header max_entry_strokes is the real limit */ +#define DICT_V4_MAX_KEY_STROKES 16 + +/* dict_v4_lookup results (>= 0); negative values are -errno */ +#define DICT_V4_MISS 0 +#define DICT_V4_FOUND_LOCAL 1 /* string id known, in *string_id */ +#define DICT_V4_FOUND_REMOTE 2 /* validx slice remote, caller RESOLVEs slot */ + +struct dict_v4_header { + uint32_t magic; + uint16_t version; + uint16_t section_count; + uint32_t n; /* union key count */ + uint32_t bucket_count; + uint32_t string_count; + uint32_t conflict_count; + uint32_t d_threshold; /* displacement escape threshold */ + uint8_t fp_bits; /* 4 */ + uint8_t dicts_mask; /* bit0 plover, bit1 lapwing */ + uint8_t max_entry_strokes; + uint8_t reserved; +} __attribute__((packed)); + +_Static_assert(sizeof(struct dict_v4_header) == 32, "header must be 32 bytes"); + +struct dict_v4_section { + uint8_t type; + uint8_t flags; + uint16_t rsvd; + uint32_t offset; /* relative to blob start */ + uint32_t len; + uint32_t param; /* VALIDX: start_slot; STRDIR: block_count; + * CONFLICTS: record count */ +} __attribute__((packed)); + +_Static_assert(sizeof(struct dict_v4_section) == 16, "dir entry must be 16 bytes"); + +struct dict_v4 { + const struct dict_v4_header *header; + + /* DISP (NULL pointers = section absent from this half) */ + const uint8_t *disp_code_len; /* u8[19] canonical code lengths */ + const uint8_t *disp_skip; /* u32 LE entries, unaligned-safe reads */ + uint32_t disp_skip_count; + const uint8_t *disp_stream; /* MSB-first Huffman bitstream */ + uint32_t disp_stream_bits; + + /* Canonical Huffman decode tables, built at init from code lengths. + * Indexed by code length (1..huff_max_len). */ + uint8_t huff_max_len; + uint8_t huff_count[DICT_V4_DISP_CLASSES + 1]; + uint32_t huff_first_code[DICT_V4_DISP_CLASSES + 1]; + uint8_t huff_sym_base[DICT_V4_DISP_CLASSES + 1]; + uint8_t huff_symbols[DICT_V4_DISP_CLASSES]; + + /* Fixed-width LSB-first packed sections */ + const uint8_t *membership; /* 2 bits/slot */ + const uint8_t *fp; /* fp_bits/slot */ + const uint8_t *validx; /* 17 bits/slot slice */ + uint32_t validx_start_slot; + uint32_t validx_slot_count; + + /* CONFLICTS: sorted 5-byte records */ + const uint8_t *conflicts; + uint32_t conflict_count; + + /* String table */ + const uint8_t *strdir_entries; /* {u32 comp_off; u32 first_string_id}[] */ + uint32_t strdir_block_count; + const uint8_t *strings; + uint32_t strings_len; +}; + +/* Parse a half blob in place; no copies. Returns 0 or -errno. */ +int dict_v4_init(struct dict_v4 *d, const void *blob, size_t len); + +/* + * Decision path (left half). Returns DICT_V4_FOUND_LOCAL (string id + * known, in *string_id), DICT_V4_FOUND_REMOTE (validx slice remote, + * caller must RESOLVE *slot), DICT_V4_MISS, or -errno. + * *slot is set on both FOUND results. + */ +int dict_v4_lookup(const struct dict_v4 *d, const uint32_t *strokes, uint8_t count, + uint8_t active_dict, uint32_t *slot, uint32_t *string_id); + +/* + * String path (right half or host test). Both return the translation + * length (>= 0, `out` NUL-terminated) or -errno. + */ +int dict_v4_string_by_id_ctx(const struct dict_v4 *d, uint32_t string_id, + char *out, size_t out_size); +int dict_v4_resolve_slot_ctx(const struct dict_v4 *d, uint32_t slot, uint8_t dict, + char *out, size_t out_size); + +uint8_t dict_v4_max_strokes(const struct dict_v4 *d); + +#ifdef __ZEPHYR__ +/* + * Singleton wrappers over the blob embedded by dict_embed.S + * (steno_dict_start/steno_dict_end): left half gets steno_v4_left.bin, + * right half steno_v4_right.bin. Lazily initialized on first use. + */ +int dict_v4_string_by_id(uint32_t string_id, char *out, size_t out_size); +int dict_v4_resolve_slot(uint32_t slot, uint8_t dict, char *out, size_t out_size); + +/* Dictionary dispatcher consumed by behavior_steno.c */ +int steno_dict_lookup(const uint32_t *strokes, uint8_t count, + char *out, size_t out_size); +uint8_t steno_dict_max_strokes(void); +void steno_dict_set_active(uint8_t dict_id); +#endif /* __ZEPHYR__ */ + +#endif /* DICT_V4_H */ diff --git a/src/split_dict.c b/src/split_dict.c index 580ddbb..f2c6592 100644 --- a/src/split_dict.c +++ b/src/split_dict.c @@ -8,214 +8,150 @@ * https://polyformproject.org/licenses/noncommercial/1.0.0 */ +#include #include #include #include #include #include #include +#include #include #include "split_dict.h" #include "split_cache.h" -#include "dict_mphf.h" LOG_MODULE_REGISTER(split_dict, CONFIG_STENO_SPLIT_LOG_LEVEL); +/* Peripheral-side v4 dict resolvers, implemented in dict_v4.c */ +extern int dict_v4_string_by_id(uint32_t string_id, char *out, size_t out_size); +extern int dict_v4_resolve_slot(uint32_t slot, uint8_t dict, char *out, size_t out_size); + /* Semaphore for blocking on BLE response */ static K_SEM_DEFINE(response_sem, 0, 1); -/* Current pending response state */ +/* Current pending response state (central side) */ static uint8_t pending_seq; -static uint8_t response_buf[256]; -static uint16_t response_len; +static uint8_t rx_buf[sizeof(struct steno_response_pkt) + SPLIT_DICT_MAX_TEXT]; +static uint16_t rx_len; static uint8_t seq_counter; +/* Notify buffer (peripheral side) */ +static uint8_t notify_buf[sizeof(struct steno_response_pkt) + SPLIT_DICT_MAX_TEXT]; + /* Cache instance */ static struct split_cache dict_cache; -/* Local MPHF dict for peripheral-side GATT lookups */ -extern const uint8_t _steno_dict_start[]; -extern const uint8_t _steno_dict_end[]; -static struct dict_mphf peripheral_mphf; -static bool peripheral_dict_ready; +/* --- Cache key synthesis --- + * + * The LRU cache keys on stroke sequences (uint32_t[]). Protocol v4 requests + * are keyed instead by string_id or (slot, dict); synthesize a two-word + * pseudo-stroke key with a tag word that can never collide with a real + * 23-bit stroke bitmask. */ -/* --- Helpers --- */ +#define CACHE_TAG_GET_STRING 0x80000010u +#define CACHE_TAG_RESOLVE 0x80000011u -static void encode_strokes(const uint32_t *strokes, uint8_t count, uint8_t *out) +static void make_string_key(uint32_t string_id, uint32_t key[2]) { - for (uint8_t i = 0; i < count; i++) { - out[i * 3 + 0] = (strokes[i] >> 16) & 0xFF; - out[i * 3 + 1] = (strokes[i] >> 8) & 0xFF; - out[i * 3 + 2] = strokes[i] & 0xFF; - } + key[0] = CACHE_TAG_GET_STRING; + key[1] = string_id; } -static void decode_strokes(const uint8_t *in, uint8_t count, uint32_t *strokes) +static void make_resolve_key(uint32_t slot, uint8_t dict, uint32_t key[2]) { - for (uint8_t i = 0; i < count; i++) { - strokes[i] = ((uint32_t)in[i * 3 + 0] << 16) | - ((uint32_t)in[i * 3 + 1] << 8) | - (uint32_t)in[i * 3 + 2]; - } + key[0] = CACHE_TAG_RESOLVE | ((uint32_t)dict << 8); + key[1] = slot; } /* --- GATT Write Callbacks (peripheral side handlers) --- */ -static ssize_t dict_query_write_cb(struct bt_conn *conn, - const struct bt_gatt_attr *attr, - const void *buf, uint16_t len, - uint16_t offset, uint8_t flags) +static void send_response(struct bt_conn *conn, const struct bt_gatt_attr *attr, + uint8_t seq, int ret, const char *text) { - const struct steno_query_pkt *pkt = buf; + struct steno_response_pkt *resp = (struct steno_response_pkt *)notify_buf; + uint16_t tlen = 0; - if (len < sizeof(struct steno_query_pkt)) { - LOG_WRN("Query pkt too short: %u", len); - return BT_GATT_ERR(BT_ATT_ERR_INVALID_ATTRIBUTE_LEN); - } - - uint8_t stroke_count = pkt->stroke_count; - uint16_t expected = sizeof(struct steno_query_pkt) + stroke_count * 3; - - if (len < expected) { - LOG_WRN("Query pkt truncated: got %u, need %u", len, expected); - return BT_GATT_ERR(BT_ATT_ERR_INVALID_ATTRIBUTE_LEN); - } - - uint32_t strokes[8]; - if (stroke_count > 8) { - stroke_count = 8; - } - decode_strokes(pkt->strokes, stroke_count, strokes); - - /* Build response */ - struct steno_response_pkt *resp = (struct steno_response_pkt *)response_buf; resp->msg_type = STENO_MSG_RESPONSE; - resp->seq = pkt->seq; + resp->seq = seq; - const char *translation = NULL; - if (peripheral_dict_ready) { - translation = dict_mphf_lookup(&peripheral_mphf, strokes, stroke_count); - } - - if (translation) { - uint16_t tlen = (uint16_t)strlen(translation); - resp->status = STENO_STATUS_FOUND; - resp->data_len = tlen; - memcpy(resp->data, translation, tlen); - response_len = sizeof(struct steno_response_pkt) + tlen; - } else { + if (ret >= 0) { + resp->status = STENO_STATUS_OK; + tlen = (uint16_t)strlen(text); + memcpy(resp->text, text, tlen); + } else if (ret == -ENOENT) { resp->status = STENO_STATUS_NOT_FOUND; - resp->data_len = 0; - response_len = sizeof(struct steno_response_pkt); + } else { + resp->status = STENO_STATUS_ERROR; } - /* Notify central with response */ - bt_gatt_notify(conn, attr, response_buf, response_len); + resp->len = sys_cpu_to_le16(tlen); + + bt_gatt_notify(conn, attr, notify_buf, sizeof(*resp) + tlen); +} + +static ssize_t handle_get_string(struct bt_conn *conn, + const struct bt_gatt_attr *attr, + const void *buf, uint16_t len) +{ + const struct steno_get_string_pkt *pkt = buf; + char text[SPLIT_DICT_MAX_TEXT + 1]; + + if (len < sizeof(*pkt)) { + LOG_WRN("GET_STRING pkt too short: %u", len); + return BT_GATT_ERR(BT_ATT_ERR_INVALID_ATTRIBUTE_LEN); + } + + uint32_t string_id = sys_le32_to_cpu(pkt->string_id); + + int ret = dict_v4_string_by_id(string_id, text, sizeof(text)); + + send_response(conn, attr, pkt->seq, ret, text); return len; } -static ssize_t dict_prefix_write_cb(struct bt_conn *conn, - const struct bt_gatt_attr *attr, - const void *buf, uint16_t len, - uint16_t offset, uint8_t flags) +static ssize_t handle_resolve(struct bt_conn *conn, + const struct bt_gatt_attr *attr, + const void *buf, uint16_t len) { - const struct steno_query_pkt *pkt = buf; + const struct steno_resolve_pkt *pkt = buf; + char text[SPLIT_DICT_MAX_TEXT + 1]; - if (len < sizeof(struct steno_query_pkt)) { + if (len < sizeof(*pkt)) { + LOG_WRN("RESOLVE pkt too short: %u", len); return BT_GATT_ERR(BT_ATT_ERR_INVALID_ATTRIBUTE_LEN); } - uint8_t stroke_count = pkt->stroke_count; - if (stroke_count > 8) { - stroke_count = 8; - } + uint32_t slot = sys_le32_to_cpu(pkt->slot); - uint32_t strokes[8]; - decode_strokes(pkt->strokes, stroke_count, strokes); + int ret = dict_v4_resolve_slot(slot, pkt->dict, text, sizeof(text)); - struct steno_response_pkt *resp = (struct steno_response_pkt *)response_buf; - resp->msg_type = STENO_MSG_RESPONSE; - resp->seq = pkt->seq; - resp->data_len = 0; - - if (peripheral_dict_ready && - stroke_count == 1 && - dict_mphf_has_prefix(&peripheral_mphf, strokes[0])) { - resp->status = STENO_STATUS_PREFIX_ONLY; - } else { - resp->status = STENO_STATUS_NOT_FOUND; - } - - response_len = sizeof(struct steno_response_pkt); - bt_gatt_notify(conn, attr, response_buf, response_len); + send_response(conn, attr, pkt->seq, ret, text); return len; } -static ssize_t dict_batch_write_cb(struct bt_conn *conn, - const struct bt_gatt_attr *attr, - const void *buf, uint16_t len, - uint16_t offset, uint8_t flags) +static ssize_t dict_req_write_cb(struct bt_conn *conn, + const struct bt_gatt_attr *attr, + const void *buf, uint16_t len, + uint16_t offset, uint8_t flags) { - const struct steno_batch_query_pkt *pkt = buf; + const uint8_t *bytes = buf; - if (len < sizeof(struct steno_batch_query_pkt)) { + if (len < 1) { return BT_GATT_ERR(BT_ATT_ERR_INVALID_ATTRIBUTE_LEN); } - LOG_DBG("Batch query: %u queries", pkt->query_count); - - /* Process each sub-query packed in queries[] */ - uint16_t pos = 0; - const uint8_t *data = pkt->queries; - uint16_t data_len = len - sizeof(struct steno_batch_query_pkt); - - for (uint8_t q = 0; q < pkt->query_count && pos < data_len; q++) { - if (pos >= data_len) { - break; - } - uint8_t stroke_count = data[pos]; - pos++; - - if (stroke_count > 8) { - stroke_count = 8; - } - if (pos + stroke_count * 3 > data_len) { - break; - } - - uint32_t strokes[8]; - decode_strokes(&data[pos], stroke_count, strokes); - pos += stroke_count * 3; - - /* Lookup and send individual response per query */ - struct steno_response_pkt *resp = (struct steno_response_pkt *)response_buf; - resp->msg_type = STENO_MSG_RESPONSE; - resp->seq = pkt->seq; - - const char *translation = NULL; - if (peripheral_dict_ready) { - translation = dict_mphf_lookup(&peripheral_mphf, strokes, stroke_count); - } - - if (translation) { - uint16_t tlen = (uint16_t)strlen(translation); - resp->status = STENO_STATUS_FOUND; - resp->data_len = tlen; - memcpy(resp->data, translation, tlen); - response_len = sizeof(struct steno_response_pkt) + tlen; - } else { - resp->status = STENO_STATUS_NOT_FOUND; - resp->data_len = 0; - response_len = sizeof(struct steno_response_pkt); - } - - bt_gatt_notify(conn, attr, response_buf, response_len); + switch (bytes[0]) { + case STENO_MSG_GET_STRING: + return handle_get_string(conn, attr, buf, len); + case STENO_MSG_RESOLVE: + return handle_resolve(conn, attr, buf, len); + default: + LOG_WRN("Unknown msg type: 0x%02x", bytes[0]); + return BT_GATT_ERR(BT_ATT_ERR_NOT_SUPPORTED); } - - return len; } /* --- Notification callback (central side) --- */ @@ -236,9 +172,12 @@ static uint8_t notify_cb(struct bt_conn *conn, return BT_GATT_ITER_CONTINUE; } - if (resp->seq == pending_seq) { - memcpy(response_buf, data, length); - response_len = length; + if (resp->msg_type == STENO_MSG_RESPONSE && resp->seq == pending_seq) { + if (length > sizeof(rx_buf)) { + length = sizeof(rx_buf); + } + memcpy(rx_buf, data, length); + rx_len = length; k_sem_give(&response_sem); } @@ -250,25 +189,11 @@ static uint8_t notify_cb(struct bt_conn *conn, BT_GATT_SERVICE_DEFINE(steno_dict_svc, BT_GATT_PRIMARY_SERVICE(STENO_UUID_SERVICE), - /* Dict Query characteristic: write + notify */ - BT_GATT_CHARACTERISTIC(STENO_UUID_DICT_QUERY, + /* Dict request characteristic: write + notify */ + BT_GATT_CHARACTERISTIC(STENO_UUID_DICT_REQ, BT_GATT_CHRC_WRITE | BT_GATT_CHRC_NOTIFY, BT_GATT_PERM_WRITE, - NULL, dict_query_write_cb, NULL), - BT_GATT_CCC(NULL, BT_GATT_PERM_READ | BT_GATT_PERM_WRITE), - - /* Dict Prefix characteristic: write + notify */ - BT_GATT_CHARACTERISTIC(STENO_UUID_DICT_PREFIX, - BT_GATT_CHRC_WRITE | BT_GATT_CHRC_NOTIFY, - BT_GATT_PERM_WRITE, - NULL, dict_prefix_write_cb, NULL), - BT_GATT_CCC(NULL, BT_GATT_PERM_READ | BT_GATT_PERM_WRITE), - - /* Dict Batch characteristic: write + notify */ - BT_GATT_CHARACTERISTIC(STENO_UUID_DICT_BATCH, - BT_GATT_CHRC_WRITE | BT_GATT_CHRC_NOTIFY, - BT_GATT_PERM_WRITE, - NULL, dict_batch_write_cb, NULL), + NULL, dict_req_write_cb, NULL), BT_GATT_CCC(NULL, BT_GATT_PERM_READ | BT_GATT_PERM_WRITE), ); @@ -278,183 +203,137 @@ BT_GATT_SERVICE_DEFINE(steno_dict_svc, static struct bt_conn *split_conn; static struct bt_gatt_subscribe_params subscribe_params; -int split_dict_lookup(const uint32_t *strokes, uint8_t count, - char *result, size_t result_size) +/* Send one request packet, block for its RESPONSE, copy text into out. + * Returns text length on success, -ENOENT on NOT_FOUND, negative errno + * otherwise. out must hold SPLIT_DICT_MAX_TEXT + 1 bytes. */ +static int split_request(const void *pkt, uint16_t pkt_len, uint8_t seq, char *out) { - if (count == 0 || count > 8) { - return -EINVAL; - } - - /* Check cache first */ - bool has_prefix; - if (split_cache_lookup(&dict_cache, strokes, count, result, result_size, &has_prefix)) { - LOG_DBG("Cache hit for %u strokes", count); - return strlen(result); - } + int err; if (!split_conn) { LOG_ERR("No split connection"); return -ENOTCONN; } - /* Build query packet */ - uint8_t pkt_buf[sizeof(struct steno_query_pkt) + 8 * 3]; - struct steno_query_pkt *pkt = (struct steno_query_pkt *)pkt_buf; - - pkt->msg_type = STENO_MSG_QUERY; - pkt->seq = seq_counter++; - pkt->stroke_count = count; - encode_strokes(strokes, count, pkt->strokes); - - pending_seq = pkt->seq; + pending_seq = seq; k_sem_reset(&response_sem); - uint16_t pkt_len = sizeof(struct steno_query_pkt) + count * 3; - - /* Send via GATT write */ - int err = bt_gatt_write_without_response(split_conn, 0, pkt_buf, pkt_len, false); + err = bt_gatt_write_without_response(split_conn, 0, pkt, pkt_len, false); if (err) { LOG_ERR("GATT write failed: %d", err); return err; } - /* Wait for response */ err = k_sem_take(&response_sem, K_MSEC(CONFIG_STENO_SPLIT_TIMEOUT_MS)); if (err) { LOG_WRN("Response timeout"); return -ETIMEDOUT; } - /* Decode response */ - const struct steno_response_pkt *resp = (const struct steno_response_pkt *)response_buf; + const struct steno_response_pkt *resp = (const struct steno_response_pkt *)rx_buf; - if (resp->status == STENO_STATUS_FOUND) { - uint16_t copy_len = resp->data_len; - if (copy_len >= result_size) { - copy_len = result_size - 1; - } - memcpy(result, resp->data, copy_len); - result[copy_len] = '\0'; - - /* Cache the result */ - split_cache_insert(&dict_cache, strokes, count, result, false); - - return copy_len; + if (resp->status == STENO_STATUS_NOT_FOUND) { + return -ENOENT; + } + if (resp->status != STENO_STATUS_OK) { + return -EIO; } - return -ENOENT; + uint16_t tlen = sys_le16_to_cpu(resp->len); + uint16_t avail = rx_len - sizeof(struct steno_response_pkt); + + if (tlen > avail) { + LOG_WRN("Response truncated: len %u, got %u", tlen, avail); + return -EIO; + } + + memcpy(out, resp->text, tlen); + out[tlen] = '\0'; + + return tlen; } -bool split_dict_has_prefix(const uint32_t *strokes, uint8_t count) +/* Copy full text into caller buffer, truncating if needed */ +static int copy_out(const char *text, uint16_t tlen, char *out, size_t out_size) { - if (count == 0 || count > 8) { - return false; + size_t copy_len = tlen; + + if (copy_len >= out_size) { + copy_len = out_size - 1; } + memcpy(out, text, copy_len); + out[copy_len] = '\0'; - /* Check cache */ - bool has_prefix; - char dummy[1]; - if (split_cache_lookup(&dict_cache, strokes, count, dummy, sizeof(dummy), &has_prefix)) { - return has_prefix; - } - - if (!split_conn) { - return false; - } - - uint8_t pkt_buf[sizeof(struct steno_query_pkt) + 8 * 3]; - struct steno_query_pkt *pkt = (struct steno_query_pkt *)pkt_buf; - - pkt->msg_type = STENO_MSG_PREFIX; - pkt->seq = seq_counter++; - pkt->stroke_count = count; - encode_strokes(strokes, count, pkt->strokes); - - pending_seq = pkt->seq; - k_sem_reset(&response_sem); - - uint16_t pkt_len = sizeof(struct steno_query_pkt) + count * 3; - - int err = bt_gatt_write_without_response(split_conn, 0, pkt_buf, pkt_len, false); - if (err) { - return false; - } - - err = k_sem_take(&response_sem, K_MSEC(CONFIG_STENO_SPLIT_TIMEOUT_MS)); - if (err) { - return false; - } - - const struct steno_response_pkt *resp = (const struct steno_response_pkt *)response_buf; - return resp->status == STENO_STATUS_PREFIX_ONLY; + return (int)copy_len; } -int split_dict_batch_lookup(const uint32_t **stroke_seqs, const uint8_t *counts, - uint8_t num_queries, struct steno_batch_result *results) +int split_dict_get_string(uint32_t string_id, char *out, size_t out_size) { - if (num_queries == 0 || !split_conn) { + if (!out || out_size == 0) { return -EINVAL; } - /* Build batch packet */ - uint8_t pkt_buf[256]; - struct steno_batch_query_pkt *pkt = (struct steno_batch_query_pkt *)pkt_buf; + uint32_t key[2]; + make_string_key(string_id, key); - pkt->msg_type = STENO_MSG_BATCH; - pkt->seq = seq_counter++; - pkt->query_count = num_queries; - - uint16_t pos = 0; - for (uint8_t q = 0; q < num_queries; q++) { - uint8_t cnt = counts[q]; - if (cnt > 8) { - cnt = 8; - } - pkt->queries[pos] = cnt; - pos++; - encode_strokes(stroke_seqs[q], cnt, &pkt->queries[pos]); - pos += cnt * 3; + if (split_cache_lookup(&dict_cache, key, 2, out, out_size, NULL)) { + LOG_DBG("Cache hit for string %u", string_id); + return strlen(out); } - uint16_t pkt_len = sizeof(struct steno_batch_query_pkt) + pos; + struct steno_get_string_pkt pkt = { + .msg_type = STENO_MSG_GET_STRING, + .seq = seq_counter++, + .string_id = sys_cpu_to_le32(string_id), + }; - pending_seq = pkt->seq; - k_sem_reset(&response_sem); - - int err = bt_gatt_write_without_response(split_conn, 0, pkt_buf, pkt_len, false); - if (err) { - return err; + char text[SPLIT_DICT_MAX_TEXT + 1]; + int ret = split_request(&pkt, sizeof(pkt), pkt.seq, text); + if (ret < 0) { + return ret; } - /* Collect responses for each query */ - for (uint8_t q = 0; q < num_queries; q++) { - err = k_sem_take(&response_sem, K_MSEC(CONFIG_STENO_SPLIT_TIMEOUT_MS)); - if (err) { - results[q].status = STENO_STATUS_ERROR; - continue; - } - - const struct steno_response_pkt *resp = - (const struct steno_response_pkt *)response_buf; - - results[q].status = resp->status; - results[q].has_prefix = (resp->status == STENO_STATUS_PREFIX_ONLY); - - if (resp->status == STENO_STATUS_FOUND && resp->data_len > 0) { - uint16_t copy_len = resp->data_len; - if (copy_len >= sizeof(results[q].translation)) { - copy_len = sizeof(results[q].translation) - 1; - } - memcpy(results[q].translation, resp->data, copy_len); - results[q].translation[copy_len] = '\0'; - results[q].translation_len = copy_len; - } else { - results[q].translation[0] = '\0'; - results[q].translation_len = 0; - } + /* Cache only entries that fit the cache value slot untruncated */ + if ((size_t)ret < SPLIT_CACHE_VALUE_SIZE) { + split_cache_insert(&dict_cache, key, 2, text, false); } - return 0; + return copy_out(text, ret, out, out_size); +} + +int split_dict_resolve(uint32_t slot, uint8_t dict, char *out, size_t out_size) +{ + if (!out || out_size == 0) { + return -EINVAL; + } + + uint32_t key[2]; + make_resolve_key(slot, dict, key); + + if (split_cache_lookup(&dict_cache, key, 2, out, out_size, NULL)) { + LOG_DBG("Cache hit for slot %u dict %u", slot, dict); + return strlen(out); + } + + struct steno_resolve_pkt pkt = { + .msg_type = STENO_MSG_RESOLVE, + .seq = seq_counter++, + .slot = sys_cpu_to_le32(slot), + .dict = dict, + }; + + char text[SPLIT_DICT_MAX_TEXT + 1]; + int ret = split_request(&pkt, sizeof(pkt), pkt.seq, text); + if (ret < 0) { + return ret; + } + + /* Cache only entries that fit the cache value slot untruncated */ + if ((size_t)ret < SPLIT_CACHE_VALUE_SIZE) { + split_cache_insert(&dict_cache, key, 2, text, false); + } + + return copy_out(text, ret, out, out_size); } int split_dict_init(void) @@ -462,20 +341,10 @@ int split_dict_init(void) split_cache_init(&dict_cache); seq_counter = 0; split_conn = NULL; + subscribe_params.notify = notify_cb; + subscribe_params.value = BT_GATT_CCC_NOTIFY; - /* Init peripheral-side MPHF dict for GATT lookups */ - size_t dict_size = _steno_dict_end - _steno_dict_start; - if (dict_size > 4) { - int ret = dict_mphf_init(&peripheral_mphf, _steno_dict_start, dict_size); - if (ret == 0) { - peripheral_dict_ready = true; - LOG_INF("Peripheral partition loaded (%u bytes)", (unsigned)dict_size); - } else { - LOG_ERR("Peripheral partition init failed: %d", ret); - } - } - - LOG_INF("Split dict initialized"); + LOG_INF("Split dict initialized (protocol v4)"); return 0; } diff --git a/src/split_dict.h b/src/split_dict.h index b2bff3a..31e7f1b 100644 --- a/src/split_dict.h +++ b/src/split_dict.h @@ -11,7 +11,6 @@ #ifndef SPLIT_DICT_H #define SPLIT_DICT_H -#include #include #include #include @@ -26,72 +25,53 @@ BT_UUID_DECLARE_128(BT_UUID_128_ENCODE( \ 0x7374656e, 0x6f00, 0x4000, 0x8000, 0x000000000001)) -#define STENO_UUID_DICT_QUERY \ +#define STENO_UUID_DICT_REQ \ BT_UUID_DECLARE_128(BT_UUID_128_ENCODE( \ 0x7374656e, 0x6f00, 0x4000, 0x8000, 0x000000000002)) -#define STENO_UUID_DICT_PREFIX \ - BT_UUID_DECLARE_128(BT_UUID_128_ENCODE( \ - 0x7374656e, 0x6f00, 0x4000, 0x8000, 0x000000000003)) - -#define STENO_UUID_DICT_BATCH \ - BT_UUID_DECLARE_128(BT_UUID_128_ENCODE( \ - 0x7374656e, 0x6f00, 0x4000, 0x8000, 0x000000000004)) - -/* Message types */ +/* BLE protocol v4 message types (FORMAT_V4.md section 7) */ enum steno_msg_type { - STENO_MSG_QUERY = 0x01, - STENO_MSG_PREFIX = 0x02, - STENO_MSG_BATCH = 0x03, - STENO_MSG_RESPONSE = 0x80, + STENO_MSG_GET_STRING = 0x10, + STENO_MSG_RESOLVE = 0x11, + STENO_MSG_RESPONSE = 0x80, }; -/* Status codes */ +/* Response status codes */ enum steno_status { - STENO_STATUS_FOUND = 0, - STENO_STATUS_NOT_FOUND = 1, - STENO_STATUS_PREFIX_ONLY = 2, - STENO_STATUS_ERROR = 3, + STENO_STATUS_OK = 0, + STENO_STATUS_NOT_FOUND = 1, + STENO_STATUS_ERROR = 2, }; -/* Packet structures */ -struct steno_query_pkt { - uint8_t msg_type; +/* Format v4 guarantees every translation <= 255 bytes */ +#define SPLIT_DICT_MAX_TEXT 255 + +/* Packet structures; all multi-byte fields little-endian on the wire */ +struct steno_get_string_pkt { + uint8_t msg_type; /* STENO_MSG_GET_STRING */ uint8_t seq; - uint8_t stroke_count; - uint8_t strokes[]; /* 3 bytes per stroke (24-bit packed) */ -} __packed; + uint32_t string_id; /* LE */ +} __attribute__((packed)); + +struct steno_resolve_pkt { + uint8_t msg_type; /* STENO_MSG_RESOLVE */ + uint8_t seq; + uint32_t slot; /* LE */ + uint8_t dict; /* 0 = plover, 1 = lapwing */ +} __attribute__((packed)); struct steno_response_pkt { - uint8_t msg_type; + uint8_t msg_type; /* STENO_MSG_RESPONSE */ uint8_t seq; - uint8_t status; - uint16_t data_len; - uint8_t data[]; /* translation string (UTF-8, not null-terminated) */ -} __packed; + uint8_t status; /* enum steno_status */ + uint16_t len; /* LE, byte length of text[] */ + char text[]; /* translation (UTF-8, not null-terminated) */ +} __attribute__((packed)); -struct steno_batch_query_pkt { - uint8_t msg_type; - uint8_t seq; - uint8_t query_count; - uint8_t queries[]; /* packed steno_query_pkt entries (without msg_type/seq) */ -} __packed; - -/* Batch result entry */ -struct steno_batch_result { - uint8_t status; - char translation[128]; - uint16_t translation_len; - bool has_prefix; -}; - -/* API */ +/* Central-side API */ int split_dict_init(void); -int split_dict_lookup(const uint32_t *strokes, uint8_t count, - char *result, size_t result_size); -bool split_dict_has_prefix(const uint32_t *strokes, uint8_t count); -int split_dict_batch_lookup(const uint32_t **stroke_seqs, const uint8_t *counts, - uint8_t num_queries, struct steno_batch_result *results); +int split_dict_get_string(uint32_t string_id, char *out, size_t out_size); +int split_dict_resolve(uint32_t slot, uint8_t dict, char *out, size_t out_size); /* GATT service registration */ int split_dict_gatt_register(void); diff --git a/src/undo.c b/src/undo.c index 188244a..2bf0b58 100644 --- a/src/undo.c +++ b/src/undo.c @@ -58,6 +58,17 @@ const struct stroke_history_entry *steno_undo_peek(const struct stroke_history * return &hist->entries[idx]; } +const struct stroke_history_entry *steno_undo_peek_at(const struct stroke_history *hist, + uint16_t back) +{ + if (back >= hist->count) { + return NULL; + } + + uint16_t idx = (hist->head + CONFIG_STENO_HISTORY_SIZE - 1 - back) % CONFIG_STENO_HISTORY_SIZE; + return &hist->entries[idx]; +} + uint16_t steno_undo_count(const struct stroke_history *hist) { return hist->count; diff --git a/src/undo.h b/src/undo.h index 33c228c..d7328d1 100644 --- a/src/undo.h +++ b/src/undo.h @@ -38,6 +38,11 @@ struct stroke_history_entry *steno_undo_pop(struct stroke_history *hist); /* Peek at most recent entry without removing. Returns NULL if empty. */ const struct stroke_history_entry *steno_undo_peek(const struct stroke_history *hist); +/* Peek `back` entries behind the most recent (0 = most recent). + * Returns NULL if out of range. */ +const struct stroke_history_entry *steno_undo_peek_at(const struct stroke_history *hist, + uint16_t back); + /* Get current count */ uint16_t steno_undo_count(const struct stroke_history *hist); diff --git a/tests/test_dict_v4.c b/tests/test_dict_v4.c new file mode 100644 index 0000000..a41d005 --- /dev/null +++ b/tests/test_dict_v4.c @@ -0,0 +1,267 @@ +/* + * Copyright (c) 2024 zmk-steno-engine contributors + * SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + * + * Licensed under the PolyForm Noncommercial License 1.0.0; + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://polyformproject.org/licenses/noncommercial/1.0.0 + */ + +/* + * Host round-trip test for the format v4 decoder. + * + * Loads the two half blobs produced by tools/compile_v4.py plus a + * vector file of (dict_id, strokes, expected translation) records, + * then drives the real split decision path: every lookup runs on the + * LEFT context; the string is fetched from the RIGHT context via + * string_by_id (FOUND_LOCAL) or resolve_slot (FOUND_REMOTE) and + * compared byte-exact. Unknown keys must MISS, modulo the 4-bit + * fingerprint false-accept rate (asserted < 10%). + * + * test_dict_v4 + * + * Vector file layout: u32 magic 0x56543456, u32 count, then records + * { u8 dict_id; u8 stroke_count; u8 known; u8 rsvd; + * u32 strokes[stroke_count]; u16 t_len; u8 translation[t_len]; }. + */ + +#include +#include +#include +#include + +#include "dict_v4.h" + +#define VECTORS_MAGIC 0x56543456u +#define MAX_MISMATCH_PRINT 10 +#define FALSE_ACCEPT_LIMIT_PCT 10.0 + +static uint8_t *load_file(const char *path, size_t *len_out) +{ + FILE *f = fopen(path, "rb"); + if (!f) { + fprintf(stderr, "FAIL: cannot open %s\n", path); + return NULL; + } + fseek(f, 0, SEEK_END); + long len = ftell(f); + fseek(f, 0, SEEK_SET); + if (len < 0) { + fclose(f); + return NULL; + } + uint8_t *buf = malloc((size_t)len); + if (!buf || fread(buf, 1, (size_t)len, f) != (size_t)len) { + fprintf(stderr, "FAIL: cannot read %s\n", path); + free(buf); + fclose(f); + return NULL; + } + fclose(f); + *len_out = (size_t)len; + return buf; +} + +static uint32_t get_le32(const uint8_t *p) +{ + return (uint32_t)p[0] | ((uint32_t)p[1] << 8) + | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24); +} + +static uint16_t get_le16(const uint8_t *p) +{ + return (uint16_t)(p[0] | (p[1] << 8)); +} + +static void print_strokes(const uint32_t *strokes, uint8_t count) +{ + for (uint8_t i = 0; i < count; i++) { + fprintf(stderr, "%s0x%06x", i ? "/" : "", strokes[i]); + } +} + +int main(int argc, char **argv) +{ + if (argc != 4) { + fprintf(stderr, "usage: %s \n", + argv[0]); + return 2; + } + + size_t left_len, right_len, vec_len; + uint8_t *left_blob = load_file(argv[1], &left_len); + uint8_t *right_blob = load_file(argv[2], &right_len); + uint8_t *vec = load_file(argv[3], &vec_len); + if (!left_blob || !right_blob || !vec) { + return 2; + } + + struct dict_v4 left, right; + int ret = dict_v4_init(&left, left_blob, left_len); + if (ret != 0) { + fprintf(stderr, "FAIL: dict_v4_init(left) = %d\n", ret); + return 1; + } + ret = dict_v4_init(&right, right_blob, right_len); + if (ret != 0) { + fprintf(stderr, "FAIL: dict_v4_init(right) = %d\n", ret); + return 1; + } + + if (left.header->n != right.header->n || + left.header->string_count != right.header->string_count || + left.header->conflict_count != right.header->conflict_count) { + fprintf(stderr, "FAIL: left/right header mismatch\n"); + return 1; + } + if (left.validx_start_slot != 0 || + left.validx_slot_count != right.validx_start_slot || + right.validx_start_slot + right.validx_slot_count != left.header->n) { + fprintf(stderr, "FAIL: VALIDX slices do not cover [0, n) " + "(left [%u, %u), right [%u, %u))\n", + left.validx_start_slot, + left.validx_start_slot + left.validx_slot_count, + right.validx_start_slot, + right.validx_start_slot + right.validx_slot_count); + return 1; + } + if (dict_v4_max_strokes(&left) == 0) { + fprintf(stderr, "FAIL: max_strokes = 0\n"); + return 1; + } + + if (vec_len < 8 || get_le32(vec) != VECTORS_MAGIC) { + fprintf(stderr, "FAIL: bad vector file magic\n"); + return 2; + } + uint32_t vec_count = get_le32(vec + 4); + + uint32_t known_checked = 0, unknown_checked = 0; + uint32_t mismatches = 0, false_accepts = 0; + uint32_t local_hits = 0, remote_hits = 0; + size_t pos = 8; + + for (uint32_t rec = 0; rec < vec_count; rec++) { + if (pos + 4 > vec_len) { + fprintf(stderr, "FAIL: truncated vector file (record %u)\n", rec); + return 2; + } + uint8_t dict_id = vec[pos]; + uint8_t stroke_count = vec[pos + 1]; + uint8_t known = vec[pos + 2]; + pos += 4; + + if (stroke_count == 0 || stroke_count > DICT_V4_MAX_KEY_STROKES || + pos + (size_t)stroke_count * 4 + 2 > vec_len) { + fprintf(stderr, "FAIL: bad vector record %u\n", rec); + return 2; + } + + uint32_t strokes[DICT_V4_MAX_KEY_STROKES]; + for (uint8_t i = 0; i < stroke_count; i++) { + strokes[i] = get_le32(vec + pos + (size_t)i * 4); + } + pos += (size_t)stroke_count * 4; + + uint16_t t_len = get_le16(vec + pos); + pos += 2; + if (pos + t_len > vec_len) { + fprintf(stderr, "FAIL: truncated translation (record %u)\n", rec); + return 2; + } + const uint8_t *expected = vec + pos; + pos += t_len; + + /* Decision path always on the LEFT context */ + uint32_t slot = 0, string_id = 0; + ret = dict_v4_lookup(&left, strokes, stroke_count, dict_id, + &slot, &string_id); + + char text[DICT_V4_MAX_TEXT + 1]; + int text_len = -1; + + /* String path on the RIGHT context */ + if (ret == DICT_V4_FOUND_LOCAL) { + local_hits++; + text_len = dict_v4_string_by_id_ctx(&right, string_id, + text, sizeof(text)); + } else if (ret == DICT_V4_FOUND_REMOTE) { + remote_hits++; + text_len = dict_v4_resolve_slot_ctx(&right, slot, dict_id, + text, sizeof(text)); + } + + if (known) { + known_checked++; + int ok = (ret == DICT_V4_FOUND_LOCAL || + ret == DICT_V4_FOUND_REMOTE) && + text_len == (int)t_len && + memcmp(text, expected, t_len) == 0; + if (!ok) { + mismatches++; + if (mismatches <= MAX_MISMATCH_PRINT) { + fprintf(stderr, "MISMATCH dict=%u strokes=", dict_id); + print_strokes(strokes, stroke_count); + fprintf(stderr, " lookup=%d text_len=%d expected=%.*s", + ret, text_len, (int)t_len, (const char *)expected); + if (text_len >= 0) { + fprintf(stderr, " got=%.*s", text_len, text); + } + fprintf(stderr, "\n"); + } + } + } else { + unknown_checked++; + if (ret == DICT_V4_FOUND_LOCAL || ret == DICT_V4_FOUND_REMOTE) { + /* Fingerprint false accept: string fetch must still + * succeed (no crash, valid id), the text is garbage. */ + false_accepts++; + if (text_len < 0) { + mismatches++; + if (mismatches <= MAX_MISMATCH_PRINT) { + fprintf(stderr, "BAD FALSE-ACCEPT dict=%u lookup=%d " + "string fetch=%d strokes=", + dict_id, ret, text_len); + print_strokes(strokes, stroke_count); + fprintf(stderr, "\n"); + } + } + } else if (ret != DICT_V4_MISS) { + mismatches++; + if (mismatches <= MAX_MISMATCH_PRINT) { + fprintf(stderr, "BAD MISS ret=%d strokes=", ret); + print_strokes(strokes, stroke_count); + fprintf(stderr, "\n"); + } + } + } + } + + double fa_pct = unknown_checked + ? 100.0 * false_accepts / unknown_checked : 0.0; + + printf("vectors: %u (known %u, unknown %u)\n", + vec_count, known_checked, unknown_checked); + printf("hits: local %u, remote %u; mismatches %u\n", + local_hits, remote_hits, mismatches); + printf("false accepts: %u / %u (%.2f%%)\n", + false_accepts, unknown_checked, fa_pct); + + if (mismatches != 0) { + printf("FAIL: %u mismatches\n", mismatches); + return 1; + } + if (fa_pct >= FALSE_ACCEPT_LIMIT_PCT) { + printf("FAIL: false-accept rate %.2f%% >= %.1f%%\n", + fa_pct, FALSE_ACCEPT_LIMIT_PCT); + return 1; + } + + printf("PASS\n"); + + free(left_blob); + free(right_blob); + free(vec); + return 0; +} diff --git a/tools/compile_v4.py b/tools/compile_v4.py new file mode 100644 index 0000000..86686ed --- /dev/null +++ b/tools/compile_v4.py @@ -0,0 +1,1353 @@ +#!/usr/bin/env python3 +# Copyright (c) 2024 zmk-steno-engine contributors +# SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 +# +# Licensed under the PolyForm Noncommercial License 1.0.0; +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# https://polyformproject.org/licenses/noncommercial/1.0.0 +"""Steno dictionary v4 "Union Split-Section" compiler. + +Implements docs/FORMAT_V4.md byte-exactly: + +- Union CHD MPHF (displacement escape, D_THRESHOLD=32768) over both dicts. +- Canonical-Huffman-coded displacements (MSB-first) with a skip table. +- MEMBERSHIP / VALIDX / FP packed LSB-first; VALIDX split across the two + half blobs at slot k (smallest k such that the right blob fits). +- Shared string table: sorted unique translations, front-coded 16KB blocks, + raw deflate (wbits=-15, level 9). +- Mandatory verification: cold-reopen both blobs and run the full decoder + path for every entry of both source dicts, byte-exact, plus 10,000 + random unknown-key probes. + +Entries never get trimmed. Anything that does not fit is a hard error. +""" + +import argparse +import heapq +import json +import math +import os +import random +import struct +import sys +import zlib +from collections import Counter, defaultdict + +try: + import numpy as np +except ImportError: + np = None + +# ─── Format constants (FORMAT_V4.md) ─── + +MAGIC = 0x344E5453 # "STN4" +VERSION = 4 +D_THRESHOLD = 32768 +SINGLETON_TRIES = 1000 +FP_BITS = 4 +VALIDX_BITS = 17 +FC_BLOCK_BOUND = 16384 +# FORMAT_V4.md section 1: spec'd 8, but longer entries exist (plover has 10 +# entries up to 11 strokes) and the spec mandates "raise max, never drop". +MAX_ENTRY_STROKES = 11 +MAX_TRANSLATION_BYTES = 255 +MAX_N = 1 << 18 # slot must fit 18 bits +MAX_STRING_ID = 1 << 17 # string_id must fit 17 bits +DISP_CLASSES = 19 + +SEC_DISP = 1 +SEC_MEMBERSHIP = 2 +SEC_VALIDX = 3 +SEC_CONFLICTS = 4 +SEC_FP = 5 +SEC_STRDIR = 6 +SEC_STRINGS = 7 + +SEC_NAMES = { + SEC_DISP: 'DISP', + SEC_MEMBERSHIP: 'MEMBERSHIP', + SEC_VALIDX: 'VALIDX', + SEC_CONFLICTS: 'CONFLICTS', + SEC_FP: 'FP', + SEC_STRDIR: 'STRDIR', + SEC_STRINGS: 'STRINGS', +} + +FNV_PRIME = 0x01000193 +FNV_OFFSET = 0x811c9dc5 +MASK32 = 0xFFFFFFFF + +DICT_PLOVER = 0 +DICT_LAPWING = 1 + +# ─── Steno stroke parsing (identical to compile_mphf.py) ─── + +STENO_KEYS = { + '#': 0x00400000, + 'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004, + 'P-': 0x00000008, 'W-': 0x00000010, 'H-': 0x00000020, + 'R-': 0x00000040, 'A-': 0x00000080, 'O-': 0x00000100, + '*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800, + '-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000, + '-B': 0x00008000, '-L': 0x00010000, '-G': 0x00020000, + '-T': 0x00040000, '-S': 0x00080000, '-D': 0x00100000, + '-Z': 0x00200000, +} + +IMPLICIT_HYPHEN = set('AOEU*') + + +def parse_stroke(s): + result = 0 + if '#' in s: + result |= STENO_KEYS['#'] + s = s.replace('#', '') + has_hyphen = '-' in s + s_clean = s.replace('-', '') + if not has_hyphen and not any(c in IMPLICIT_HYPHEN for c in s_clean): + for c in s_clean: + key = c + '-' + if key in STENO_KEYS: + result |= STENO_KEYS[key] + return result + if has_hyphen: + hyphen_pos = s.index('-') + for i, c in enumerate(s): + if c == '-': + continue + if c in 'AO': + result |= STENO_KEYS[c + '-'] + elif c in 'EU': + result |= STENO_KEYS['-' + c] + elif c == '*': + result |= STENO_KEYS['*'] + elif i < hyphen_pos and (c + '-') in STENO_KEYS: + result |= STENO_KEYS[c + '-'] + elif i > hyphen_pos and ('-' + c) in STENO_KEYS: + result |= STENO_KEYS['-' + c] + else: + past_vowels = False + for c in s_clean: + if c in 'AO': + result |= STENO_KEYS[c + '-'] + past_vowels = True + elif c in 'EU': + result |= STENO_KEYS['-' + c] + past_vowels = True + elif c == '*': + result |= STENO_KEYS['*'] + past_vowels = True + elif not past_vowels and (c + '-') in STENO_KEYS: + result |= STENO_KEYS[c + '-'] + elif past_vowels and ('-' + c) in STENO_KEYS: + result |= STENO_KEYS['-' + c] + elif (c + '-') in STENO_KEYS: + result |= STENO_KEYS[c + '-'] + return result + + +def encode_key(stroke_str): + """Parse stroke string → (strokes tuple, key_bytes: u32 LE per stroke).""" + parts = stroke_str.split('/') + strokes = tuple(parse_stroke(s) for s in parts) + key_bytes = b''.join(struct.pack('> i) & 1: + self.data[-1] |= (0x80 >> (self.bit_pos % 8)) + self.bit_pos += 1 + + +class MSBReader: + """MSB-first bit reader (DISP Huffman stream only).""" + + def __init__(self, data): + self.data = data + self.bit_pos = 0 + + def read_bit(self): + b = (self.data[self.bit_pos >> 3] >> (7 - (self.bit_pos & 7))) & 1 + self.bit_pos += 1 + return b + + def read_bits(self, n_bits): + v = 0 + for _ in range(n_bits): + v = (v << 1) | self.read_bit() + return v + + +def pack_lsb(values, bits): + """Pack fixed-width values LSB-first. Returns exactly ceil(m*bits/8) bytes.""" + m = len(values) + n_bytes = (m * bits + 7) // 8 + if m == 0: + return b'' + if np is not None: + v = np.asarray(values, dtype=np.uint64) + offs = np.arange(m, dtype=np.uint64) * np.uint64(bits) + byte_idx = (offs >> np.uint64(3)).astype(np.int64) + shift = offs & np.uint64(7) + shifted = v << shift + buf = np.zeros(n_bytes + 8, dtype=np.uint8) + span = (bits + 14) // 8 + for b in range(span): + np.bitwise_or.at( + buf, byte_idx + b, + ((shifted >> np.uint64(8 * b)) & np.uint64(0xFF)).astype(np.uint8)) + return bytes(buf[:n_bytes]) + w = BitWriter() + for v in values: + w.write_bits(v, bits) + return bytes(w.data[:n_bytes]) + + +def read_lsb_at(section, idx, bits): + """Scalar LSB-first fixed-width read (decoder reference path).""" + off = idx * bits + byte_idx = off >> 3 + shift = off & 7 + window = section[byte_idx:byte_idx + 4] + b'\x00' * 4 + acc = int.from_bytes(window[:4], 'little') + return (acc >> shift) & ((1 << bits) - 1) + + +def read_lsb_many(section, idxs, bits): + """Vectorized LSB-first fixed-width reads; same bit math as read_lsb_at.""" + if np is None or not hasattr(idxs, 'dtype'): + return [read_lsb_at(section, int(i), bits) for i in idxs] + if len(idxs) == 0: + return np.zeros(0, dtype=np.int64) + buf = np.frombuffer(bytes(section) + b'\x00' * 8, dtype=np.uint8) + offs = idxs.astype(np.uint64) * np.uint64(bits) + byte_idx = (offs >> np.uint64(3)).astype(np.int64) + shift = offs & np.uint64(7) + acc = np.zeros(len(idxs), dtype=np.uint64) + span = (bits + 14) // 8 + for b in range(span): + acc |= buf[byte_idx + b].astype(np.uint64) << np.uint64(8 * b) + return ((acc >> shift) & np.uint64((1 << bits) - 1)).astype(np.int64) + + +# ─── Vectorized FNV-1a helpers ─── + +def _fold_mat(h, mat): + for j in range(mat.shape[1]): + h = ((h ^ mat[:, j]) * FNV_PRIME) & MASK32 + return h + + +def _fold_seeds(seeds): + h = np.full(seeds.shape, FNV_OFFSET, dtype=np.uint64) + for shift in (0, 8, 16, 24): + h = ((h ^ ((seeds >> np.uint64(shift)) & np.uint64(0xFF))) * FNV_PRIME) & MASK32 + return h + + +def _group_by_len(keys, idxs=None): + by_len = defaultdict(list) + if idxs is None: + for i, kb in enumerate(keys): + by_len[len(kb)].append(i) + else: + for i in idxs: + by_len[len(keys[i])].append(int(i)) + return by_len + + +def _key_matrix(keys, idxs, length): + return np.frombuffer(b''.join(keys[i] for i in idxs), dtype=np.uint8) \ + .reshape(len(idxs), length).astype(np.uint64) + + +def hash0_all(keys): + """hash_key(kb, 0) for all keys.""" + if np is None: + return [hash_key(kb, 0) for kb in keys] + out = np.zeros(len(keys), dtype=np.uint64) + for length, idxs in _group_by_len(keys).items(): + ia = np.array(idxs, dtype=np.int64) + h = np.full(len(idxs), FNV_OFFSET, dtype=np.uint64) + for _ in range(4): # seed 0 → four 0x00 bytes + h = (h * FNV_PRIME) & MASK32 + out[ia] = _fold_mat(h, _key_matrix(keys, idxs, length)) + return out + + +def fp_all(keys): + """fnv1a(kb) & 0xF for all keys.""" + if np is None: + return [fnv1a_32(kb) & 0xF for kb in keys] + out = np.zeros(len(keys), dtype=np.int64) + for length, idxs in _group_by_len(keys).items(): + ia = np.array(idxs, dtype=np.int64) + h = np.full(len(idxs), FNV_OFFSET, dtype=np.uint64) + h = _fold_mat(h, _key_matrix(keys, idxs, length)) + out[ia] = (h & np.uint64(0xF)).astype(np.int64) + return out + + +def compute_slots(keys, disp, bucket_count, n): + """Decoder slot computation for a list of keys against a disp array.""" + if np is None: + slots = [] + for kb in keys: + b = hash_key(kb, 0) % bucket_count + d = disp[b] + if d >= D_THRESHOLD: + slots.append(d - D_THRESHOLD) + else: + slots.append(hash_key(kb, d + 1) % n) + return slots + disp_np = np.asarray(disp, dtype=np.int64) + h0 = hash0_all(keys) + bucket = (h0 % np.uint64(bucket_count)).astype(np.int64) + d = disp_np[bucket] + slot = np.empty(len(keys), dtype=np.int64) + direct = d >= D_THRESHOLD + slot[direct] = d[direct] - D_THRESHOLD + nd = np.nonzero(~direct)[0] + for length, idxs in _group_by_len(keys, nd).items(): + ia = np.array(idxs, dtype=np.int64) + seeds = (d[ia] + 1).astype(np.uint64) + h = _fold_seeds(seeds) + h = _fold_mat(h, _key_matrix(keys, idxs, length)) + slot[ia] = (h % np.uint64(n)).astype(np.int64) + return slot + + +# ─── Dictionary loading ─── + +def load_dict(path, name, quiet=False): + """Load a Plover-format JSON dict. + + Returns (raw_entries, kb_map, offenders) where: + raw_entries = [(stroke_str, key_bytes)] for every JSON entry + kb_map = {key_bytes: (translation_bytes, stroke_str, n_strokes)}, + FIRST occurrence wins for duplicate key bytes + offenders = list of hard-error strings (too many strokes / too long) + """ + with open(path) as f: + raw = json.load(f) + raw_entries = [] + kb_map = {} + offenders = [] + for stroke_str, translation in raw.items(): + strokes, kb = encode_key(stroke_str) + raw_entries.append((stroke_str, kb)) + if len(strokes) > MAX_ENTRY_STROKES: + offenders.append(f"{name}: '{stroke_str}' has {len(strokes)} strokes " + f"(max {MAX_ENTRY_STROKES})") + tb = translation.encode('utf-8') + if len(tb) > MAX_TRANSLATION_BYTES: + offenders.append(f"{name}: '{stroke_str}' translation is {len(tb)} bytes " + f"(max {MAX_TRANSLATION_BYTES})") + if kb in kb_map: + prev_tb, prev_stroke, _ = kb_map[kb] + if not quiet: + print(f" dup key ({name}): '{stroke_str}' → " + f"{translation!r} loses to '{prev_stroke}' → " + f"{prev_tb.decode('utf-8', 'replace')!r} (first wins)", + file=sys.stderr) + continue + kb_map[kb] = (tb, stroke_str, len(strokes)) + return raw_entries, kb_map, offenders + + +# ─── CHD MPHF construction ─── + +def build_chd_np(keys, bucket_count, n): + """Numpy-vectorized CHD with displacement escape. + + Multi-key buckets search d in [0, D_THRESHOLD). + Single-key buckets search d in [0, SINGLETON_TRIES), then direct-place + into the lowest free slot (disp = D_THRESHOLD + slot). + Returns (disp list, slot_of_key list, stats dict); hard error on failure. + """ + h0 = hash0_all(keys) + seed_hashes = _fold_seeds(np.arange(1, D_THRESHOLD + 1, dtype=np.uint64)) + + bucket_of = (h0 % np.uint64(bucket_count)).astype(np.int64) + order = np.argsort(bucket_of, kind='stable') + sorted_buckets = bucket_of[order] + uniq, starts = np.unique(sorted_buckets, return_index=True) + groups = [] + for gi in range(len(uniq)): + s = starts[gi] + e = starts[gi + 1] if gi + 1 < len(uniq) else n + groups.append((int(uniq[gi]), order[s:e])) + groups.sort(key=lambda g: -len(g[1])) + + free = np.ones(n, dtype=bool) + slot_of_key = np.full(n, -1, dtype=np.int64) + disp = np.zeros(bucket_count, dtype=np.int64) + max_hash_disp = 0 + direct_count = 0 + free_ptr = 0 + + def chunks_multi(): + yield (0, 512) + s = 512 + while s < D_THRESHOLD: + e = min(s + 8192, D_THRESHOLD) + yield (s, e) + s = e + + for bucket_id, members in groups: + m = len(members) + kbs = [keys[i] for i in members] + if m == 1: + kb = kbs[0] + h = seed_hashes[:SINGLETON_TRIES].copy() + for b in kb: + h = ((h ^ b) * FNV_PRIME) & MASK32 + slots = h % np.uint64(n) + hit = np.nonzero(free[slots])[0] + if hit.size: + d = int(hit[0]) + slot = int(slots[hit[0]]) + free[slot] = False + slot_of_key[members[0]] = slot + disp[bucket_id] = d + if d > max_hash_disp: + max_hash_disp = d + else: + while not free[free_ptr]: + free_ptr += 1 + slot = free_ptr + free[slot] = False + slot_of_key[members[0]] = slot + disp[bucket_id] = D_THRESHOLD + slot + direct_count += 1 + continue + + placed = False + for cs, ce in chunks_multi(): + base = seed_hashes[cs:ce] + slot_rows = [] + for kb in kbs: + h = base.copy() + for b in kb: + h = ((h ^ b) * FNV_PRIME) & MASK32 + slot_rows.append(h % np.uint64(n)) + slots = np.stack(slot_rows) + ok = free[slots].all(axis=0) + for i in range(m): + for j in range(i + 1, m): + ok &= slots[i] != slots[j] + hit = np.nonzero(ok)[0] + if hit.size: + d = cs + int(hit[0]) + chosen = slots[:, hit[0]] + free[chosen] = False + for i, mi in enumerate(members): + slot_of_key[mi] = int(chosen[i]) + disp[bucket_id] = d + if d > max_hash_disp: + max_hash_disp = d + placed = True + break + if not placed: + print(f"FATAL: CHD bucket {bucket_id} ({m} keys) found no displacement " + f"in [0, {D_THRESHOLD})", file=sys.stderr) + sys.exit(1) + + if not (slot_of_key >= 0).all(): + print("FATAL: CHD left keys unplaced", file=sys.stderr) + sys.exit(1) + stats = {'max_hash_disp': int(max_hash_disp), 'direct_count': int(direct_count)} + return [int(v) for v in disp], [int(s) for s in slot_of_key], stats + + +def build_chd_py(keys, bucket_count, n): + """Pure-python CHD fallback (same algorithm, no numpy).""" + buckets = defaultdict(list) + for idx, kb in enumerate(keys): + buckets[hash_key(kb, 0) % bucket_count].append(idx) + groups = sorted(buckets.items(), key=lambda g: -len(g[1])) + + free = [True] * n + slot_of_key = [-1] * n + disp = [0] * bucket_count + max_hash_disp = 0 + direct_count = 0 + free_ptr = 0 + + for bucket_id, members in groups: + m = len(members) + kbs = [keys[i] for i in members] + if m == 1: + kb = kbs[0] + placed = False + for d in range(SINGLETON_TRIES): + slot = hash_key(kb, d + 1) % n + if free[slot]: + free[slot] = False + slot_of_key[members[0]] = slot + disp[bucket_id] = d + max_hash_disp = max(max_hash_disp, d) + placed = True + break + if not placed: + while not free[free_ptr]: + free_ptr += 1 + free[free_ptr] = False + slot_of_key[members[0]] = free_ptr + disp[bucket_id] = D_THRESHOLD + free_ptr + direct_count += 1 + continue + + placed = False + for d in range(D_THRESHOLD): + slots = [] + seen = set() + ok = True + for kb in kbs: + slot = hash_key(kb, d + 1) % n + if not free[slot] or slot in seen: + ok = False + break + seen.add(slot) + slots.append(slot) + if not ok: + continue + for i, mi in enumerate(members): + free[slots[i]] = False + slot_of_key[mi] = slots[i] + disp[bucket_id] = d + max_hash_disp = max(max_hash_disp, d) + placed = True + break + if not placed: + print(f"FATAL: CHD bucket {bucket_id} ({m} keys) found no displacement " + f"in [0, {D_THRESHOLD})", file=sys.stderr) + sys.exit(1) + + if any(s < 0 for s in slot_of_key): + print("FATAL: CHD left keys unplaced", file=sys.stderr) + sys.exit(1) + stats = {'max_hash_disp': max_hash_disp, 'direct_count': direct_count} + return disp, slot_of_key, stats + + +def build_chd(keys, bucket_count, n): + if np is not None: + return build_chd_np(keys, bucket_count, n) + return build_chd_py(keys, bucket_count, n) + + +# ─── Canonical Huffman (DISP classes) ─── + +def huffman_lengths(freq): + """freq: {symbol: count} → {symbol: code length}.""" + items = sorted(freq.items()) + if len(items) == 1: + return {items[0][0]: 1} + heap = [] + uid = 0 + for sym, count in items: + heapq.heappush(heap, (count, uid, ('leaf', sym))) + uid += 1 + while len(heap) > 1: + c1, _, t1 = heapq.heappop(heap) + c2, _, t2 = heapq.heappop(heap) + heapq.heappush(heap, (c1 + c2, uid, ('node', t1, t2))) + uid += 1 + lengths = {} + stack = [(heap[0][2], 0)] + while stack: + node, depth = stack.pop() + if node[0] == 'leaf': + lengths[node[1]] = depth + else: + stack.append((node[1], depth + 1)) + stack.append((node[2], depth + 1)) + return lengths + + +def canonical_codes(lengths): + """Canonical codes in (code_len, symbol) order, numerically increasing, + MSB-first. Returns {symbol: (code, length)}.""" + syms = sorted(lengths, key=lambda s: (lengths[s], s)) + codes = {} + code = 0 + prev_len = None + for s in syms: + length = lengths[s] + if prev_len is not None: + code = (code + 1) << (length - prev_len) + codes[s] = (code, length) + prev_len = length + return codes + + +def disp_class(v): + return v.bit_length() + + +def build_disp_section(disp, bucket_count): + """DISP section: code_len[19], skip table, MSB-first Huffman stream.""" + classes = [disp_class(v) for v in disp] + max_class = max(classes) + if max_class >= DISP_CLASSES: + print(f"FATAL: displacement class {max_class} out of range " + f"(max {DISP_CLASSES - 1})", file=sys.stderr) + sys.exit(1) + lengths = huffman_lengths(Counter(classes)) + codes = canonical_codes(lengths) + + w = MSBWriter() + skips = [] + for i, v in enumerate(disp): + if i % 256 == 0: + skips.append(w.bit_pos) + c = classes[i] + code, length = codes[c] + w.write_bits(code, length) + if c >= 2: + w.write_bits(v - (1 << (c - 1)), c - 1) + + assert len(skips) == (bucket_count + 255) // 256 + code_len_bytes = bytes(lengths.get(c, 0) for c in range(DISP_CLASSES)) + section = (code_len_bytes + + struct.pack(' 24: + raise ValueError("DISP Huffman decode ran away") + if sym <= 1: + return sym + return (1 << (sym - 1)) | reader.read_bits(sym - 1) + + +def disp_random_access(stream, codemap, skips, bucket): + """Random access decode of bucket's displacement via the skip table.""" + r = MSBReader(stream) + r.bit_pos = skips[bucket >> 8] + for _ in range(bucket & 255): + decode_disp_value(r, codemap) + return decode_disp_value(r, codemap) + + +# ─── String table (front-coded deflate blocks) ─── + +def common_prefix_len(a, b): + m = min(len(a), len(b), 255) + i = 0 + while i < m and a[i] == b[i]: + i += 1 + return i + + +def build_string_sections(strings): + """strings: sorted unique translation bytes. + Returns (strdir, strings_section, block_count, raw_fc_bytes).""" + blocks = [] # (first_string_id, raw_fc_bytes) + cur = bytearray() + cur_first = 0 + prev = None + for sid, s in enumerate(strings): + if prev is None: + enc = bytes([0]) + s + b'\x00' + else: + p = common_prefix_len(prev, s) + enc = bytes([p]) + s[p:] + b'\x00' + if len(cur) + len(enc) > FC_BLOCK_BOUND: + blocks.append((cur_first, bytes(cur))) + cur = bytearray() + cur_first = sid + prev = None + enc = bytes([0]) + s + b'\x00' + cur += enc + prev = s + if cur: + blocks.append((cur_first, bytes(cur))) + + comp_blocks = [] + raw_total = 0 + for first_id, raw in blocks: + assert len(raw) <= FC_BLOCK_BOUND + raw_total += len(raw) + co = zlib.compressobj(9, zlib.DEFLATED, -15) + comp_blocks.append(co.compress(raw) + co.flush()) + + strdir = struct.pack('> 18 + if slot <= prev_slot: + print(f"VERIFY FAIL: CONFLICTS not sorted at record {i}", + file=sys.stderr) + failures += 1 + prev_slot = slot + conflict_map[slot] = sid + + strtab = StringTable(strdir_sec, strings_sec, strdir_blocks) + + k = rvalidx_start + if lvalidx_start != 0: + print("VERIFY FAIL: left VALIDX start_slot != 0", file=sys.stderr) + failures += 1 + left_k = len(lvalidx_sec) * 8 // VALIDX_BITS + if left_k != k: + print(f"VERIFY FAIL: left VALIDX holds {left_k} slots, right starts at {k}", + file=sys.stderr) + failures += 1 + + # Full decoder path for every unique key of each dict + key_results = {} # dict_id -> {kb: (ok, decoded_bytes)} + for dict_id, kb_map in dict_maps.items(): + keys = list(kb_map) + expected = [kb_map[kb][0] for kb in keys] + slots = compute_slots(keys, disp, bucket_count, n) + fp_expect = fp_all(keys) + if np is not None: + slots_np = slots if hasattr(slots, 'dtype') else np.asarray(slots) + fp_read = read_lsb_many(fp_sec, slots_np, FP_BITS) + memb_read = read_lsb_many(memb_sec, slots_np, 2) + sid_read = np.empty(len(keys), dtype=np.int64) + left_mask = slots_np < k + sid_read[left_mask] = read_lsb_many( + lvalidx_sec, slots_np[left_mask], VALIDX_BITS) + sid_read[~left_mask] = read_lsb_many( + rvalidx_sec, slots_np[~left_mask] - k, VALIDX_BITS) + else: + fp_read = [read_lsb_at(fp_sec, s, FP_BITS) for s in slots] + memb_read = [read_lsb_at(memb_sec, s, 2) for s in slots] + sid_read = [read_lsb_at(lvalidx_sec, s, VALIDX_BITS) if s < k + else read_lsb_at(rvalidx_sec, s - k, VALIDX_BITS) + for s in slots] + + results = {} + for i, kb in enumerate(keys): + slot = int(slots[i]) + ok = True + decoded = None + if not (0 <= slot < n): + ok = False + elif int(fp_read[i]) != int(fp_expect[i]): + ok = False + elif not (int(memb_read[i]) & (1 << dict_id)): + ok = False + else: + sid = int(sid_read[i]) + if dict_id == DICT_LAPWING and int(memb_read[i]) == 3: + sid = conflict_map.get(slot, sid) + decoded = strtab.get(sid) + if decoded != expected[i]: + ok = False + if not ok: + failures += 1 + if failures <= 20: + print(f"VERIFY FAIL: dict {dict_id} key " + f"'{kb_map[kb][1]}' slot {slot}: decoded " + f"{decoded!r}, expected {expected[i]!r}", + file=sys.stderr) + results[kb] = ok + key_results[dict_id] = results + + # Every raw entry of both source dicts (dup-losers resolve to first-wins) + entries_checked = 0 + for dict_id, entries in raw_entries.items(): + results = key_results[dict_id] + for _stroke_str, kb in entries: + entries_checked += 1 + if not results[kb]: + failures += 1 + + # 10k random unknown-key probes + union = set() + for kb_map in dict_maps.values(): + union.update(kb_map) + rng = random.Random(0x5634) + probes = [] + while len(probes) < 10000: + ns = rng.choice((1, 1, 2, 3)) + kb = b''.join(struct.pack('= MAX_N: + print(f"FATAL: union key count {n} >= {MAX_N} (slot must fit 18 bits)", + file=sys.stderr) + sys.exit(1) + max_entry_strokes = max( + max((v[2] for v in m.values()), default=0) for m in dict_maps.values()) + print(f"Union: n={n}, max_entry_strokes={max_entry_strokes}", file=sys.stderr) + + # Shared string table: sorted unique translations (UTF-8 byte order) + strings = sorted({v[0] for m in dict_maps.values() for v in m.values()}) + string_count = len(strings) + if string_count > MAX_STRING_ID: + print(f"FATAL: {string_count} unique strings > {MAX_STRING_ID} " + f"(string_id must fit {VALIDX_BITS} bits)", file=sys.stderr) + sys.exit(1) + string_id = {s: i for i, s in enumerate(strings)} + print(f"Strings: {string_count} unique", file=sys.stderr) + + # CHD MPHF + bucket_count = n // 4 + print(f"Building CHD: n={n}, buckets={bucket_count}, " + f"D_THRESHOLD={D_THRESHOLD}", file=sys.stderr) + disp, slot_of_key, chd_stats = build_chd(keys, bucket_count, n) + print(f" CHD ok: max_hash_disp={chd_stats['max_hash_disp']}, " + f"direct={chd_stats['direct_count']}", file=sys.stderr) + + # Per-slot tables + fps = fp_all(keys) + memb_slot = [0] * n + val_slot = [0] * n + fp_slot = [0] * n + conflicts = [] + for i, kb in enumerate(keys): + slot = slot_of_key[i] + m = ((1 if kb in plover_map else 0) + | (2 if kb in lapwing_map else 0)) + memb_slot[slot] = m + fp_slot[slot] = int(fps[i]) + if kb in plover_map: + sid = string_id[plover_map[kb][0]] + else: + sid = string_id[lapwing_map[kb][0]] + val_slot[slot] = sid + if m == 3 and plover_map[kb][0] != lapwing_map[kb][0]: + conflicts.append((slot, string_id[lapwing_map[kb][0]])) + conflicts.sort() + conflict_count = len(conflicts) + print(f"Membership: both={sum(1 for m in memb_slot if m == 3)}, " + f"conflicts={conflict_count}", file=sys.stderr) + + # Sections + disp_sec = build_disp_section(disp, bucket_count) + memb_sec = pad4(pack_lsb(memb_slot, 2)) + fp_sec = pad4(pack_lsb(fp_slot, FP_BITS)) + conf_sec = b''.join((slot | (sid << 18)).to_bytes(5, 'little') + for slot, sid in conflicts) + strdir_sec, strings_sec, block_count, raw_fc_bytes = \ + build_string_sections(strings) + print(f"Section sizes: DISP={len(disp_sec)} MEMBERSHIP={len(memb_sec)} " + f"FP={len(fp_sec)} CONFLICTS={len(conf_sec)} " + f"STRDIR={len(strdir_sec)} STRINGS={len(strings_sec)} " + f"(FC raw={raw_fc_bytes}, {block_count} blocks) " + f"VALIDX total={validx_len_bytes(n)}", file=sys.stderr) + + # Placement: k = smallest such that the right blob fits --right-size + right_fixed = [len(strdir_sec), len(strings_sec), len(conf_sec)] + + def right_size(k): + return blob_total_size(right_fixed + [validx_len_bytes(n - k)]) + + def left_size(k): + return blob_total_size([len(disp_sec), len(memb_sec), len(fp_sec), + len(conf_sec), validx_len_bytes(k)]) + + if right_size(n) > args.right_size: + print(f"FATAL: right blob without any VALIDX is {right_size(n)} bytes " + f"> budget {args.right_size}. Nothing may be trimmed.", + file=sys.stderr) + sys.exit(1) + lo, hi = 0, n + while lo < hi: + mid = (lo + hi) // 2 + if right_size(mid) <= args.right_size: + hi = mid + else: + lo = mid + 1 + k = lo + lsz, rsz = left_size(k), right_size(k) + print(f"Placement: k={k} → left={lsz} (budget {args.left_size}), " + f"right={rsz} (budget {args.right_size})", file=sys.stderr) + if lsz > args.left_size: + print(f"FATAL: left blob {lsz} bytes exceeds budget {args.left_size} " + f"(right={rsz}/{args.right_size}, k={k}, n={n}, " + f"left VALIDX={validx_len_bytes(k)}, " + f"right VALIDX={validx_len_bytes(n - k)}). " + f"Nothing may be trimmed.", file=sys.stderr) + print("RESULT_JSON: " + json.dumps({ + 'ok': False, 'k': k, 'n': n, + 'left_bytes': lsz, 'right_bytes': rsz, + 'left_budget': args.left_size, 'right_budget': args.right_size, + 'sections': { + 'DISP': len(disp_sec), 'MEMBERSHIP': len(memb_sec), + 'FP': len(fp_sec), 'CONFLICTS': len(conf_sec), + 'STRDIR': len(strdir_sec), 'STRINGS': len(strings_sec), + 'VALIDX_LEFT': validx_len_bytes(k), + 'VALIDX_RIGHT': validx_len_bytes(n - k), + }, + })) + sys.exit(1) + + validx_left = pack_lsb(val_slot[:k], VALIDX_BITS) + validx_right = pack_lsb(val_slot[k:], VALIDX_BITS) + assert len(validx_left) == validx_len_bytes(k) + assert len(validx_right) == validx_len_bytes(n - k) + + left_sections = [ + (SEC_DISP, disp_sec, 0), + (SEC_MEMBERSHIP, memb_sec, 0), + (SEC_FP, fp_sec, 0), + (SEC_CONFLICTS, conf_sec, conflict_count), + (SEC_VALIDX, validx_left, 0), + ] + right_sections = [ + (SEC_STRDIR, strdir_sec, block_count), + (SEC_STRINGS, strings_sec, 0), + (SEC_CONFLICTS, conf_sec, conflict_count), + (SEC_VALIDX, validx_right, k), + ] + + left_blob = assemble_blob(left_sections, n, bucket_count, string_count, + conflict_count, dicts_mask, max_entry_strokes) + right_blob = assemble_blob(right_sections, n, bucket_count, string_count, + conflict_count, dicts_mask, max_entry_strokes) + assert len(left_blob) == lsz and len(right_blob) == rsz + + os.makedirs(args.out_dir, exist_ok=True) + left_path = os.path.join(args.out_dir, 'steno_v4_left.bin') + right_path = os.path.join(args.out_dir, 'steno_v4_right.bin') + with open(left_path, 'wb') as f: + f.write(left_blob) + with open(right_path, 'wb') as f: + f.write(right_blob) + print(f"Wrote {left_path} ({len(left_blob)} bytes)", file=sys.stderr) + print(f"Wrote {right_path} ({len(right_blob)} bytes)", file=sys.stderr) + + # Mandatory verification (cold reopen) + print("Verifying (cold reopen, full decode path, every entry)...", + file=sys.stderr) + vres = verify_blobs(left_path, right_path, dict_maps, raw_entries, + dicts_mask) + print(f" entries checked: {vres['entries_checked']}, " + f"failures: {vres['failures']}", file=sys.stderr) + print(f" unknown probes: {vres['probes']}, " + f"false-accept {vres['false_accept_pct']:.2f}% " + f"(fp-only pass {vres['fp_pass_pct']:.2f}%)", file=sys.stderr) + if vres['failures']: + print(f"VERIFICATION FAILED: {vres['failures']} failures", + file=sys.stderr) + sys.exit(1) + + def section_report(sections, half): + out = [] + for sec_type, data, param in sections: + out.append({'name': SEC_NAMES[sec_type], 'bytes': len(data), + 'param': param, 'half': half}) + return out + + manifest = { + 'format_version': VERSION, + 'n': n, + 'bucket_count': bucket_count, + 'string_count': string_count, + 'conflict_count': conflict_count, + 'k_split_slot': k, + 'max_entry_strokes': max_entry_strokes, + 'dicts_mask': dicts_mask, + 'd_threshold': D_THRESHOLD, + 'fp_bits': FP_BITS, + 'chd': chd_stats, + 'strings': { + 'block_count': block_count, + 'raw_fc_bytes': raw_fc_bytes, + 'compressed_bytes': len(strings_sec), + }, + 'left': { + 'file': 'steno_v4_left.bin', + 'bytes': len(left_blob), + 'budget': args.left_size, + 'sections': section_report(left_sections, 'left'), + }, + 'right': { + 'file': 'steno_v4_right.bin', + 'bytes': len(right_blob), + 'budget': args.right_size, + 'sections': section_report(right_sections, 'right'), + }, + 'verification': { + 'entries_checked': vres['entries_checked'], + 'failures': vres['failures'], + 'unknown_probes': vres['probes'], + 'false_accept_pct': vres['false_accept_pct'], + 'fp_only_pass_pct': vres['fp_pass_pct'], + }, + } + manifest_path = os.path.join(args.out_dir, 'manifest.json') + with open(manifest_path, 'w') as f: + json.dump(manifest, f, indent=2) + print(f"Wrote {manifest_path}", file=sys.stderr) + + for half in ('left', 'right'): + print(f"{half}: {manifest[half]['bytes']} bytes " + f"(budget {manifest[half]['budget']})") + for s in manifest[half]['sections']: + print(f" {s['name']:<11} {s['bytes']:>8} bytes (param {s['param']})") + print("RESULT_JSON: " + json.dumps({'ok': True, 'manifest': manifest})) + + +if __name__ == '__main__': + main()