afiqzudinhadi/zmk-steno-engine
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

v4 union split-section dictionary: both full dicts, zero trimming

Browse source 
Format v4 (docs/FORMAT_V4.md): one union CHD MPHF over 226,791 stroke
keys from Plover (147,424) + Lapwing (114,885), sections distributed
across halves. Left blob 387KB (displacements log-class Huffman 59KB,
membership 57KB, fingerprints 113KB, conflicts 9.6KB, value-index
slice 157KB); right blob 515KB (strings front-coded+deflate 193KB,
value-index remainder 324KB, conflicts dup). Shared string table:
73,464 unique translations, Lapwing 93% subset of Plover.

- tools/compile_v4.py: encoder + full verification (all 262,309
  entries byte-exact through cold decode path; hard error if either
  blob exceeds budget — never trims)
- src/dict_v4.c/.h: decoder — canonical Huffman displacement decode,
  conflict binary search, FC block walk, 2-block LRU cache
- src/behavior_steno.c: Plover-style sliding longest-match with
  retrace replaces multi-stroke timeout + prefix checks
- src/split_dict.c/.h: protocol v4 — GET_STRING(string_id) /
  RESOLVE(slot, dict); decisions are 100% left-local, BLE only for
  translation text
- CMake/Kconfig: STENO_DICT_BOTH default, one compiler run emits both
  half blobs; legacy MPHF path kept behind STENO_DICT_V4=n
- Host round-trip test: 42,000 vectors left→right split path,
  0 mismatches (tests/test_dict_v4.c)
This commit is contained in:
afiqzudinhadi 2026-07-03 01:28:31 +08:00
parent dd593c1640
commit 9bf21c9db2
12 changed files with 3338 additions and 666 deletions
Download patch file Download diff file Expand all files Collapse all files

223
CMakeLists.txt
View file

@ -8,14 +8,135 @@ target_include_directories(app PRIVATE
${CMAKE_CURRENT_SOURCE_DIR}/src
)
# ── Split-dict mode ──────────────────────────────────
# TRUE split: both halves have local MPHF partition + BLE fallback
# Central: behavior engine + local left partition + BLE client
# Peripheral: local right partition + GATT server
if(CONFIG_STENO_SPLIT_DICT)
find_package(Python3 REQUIRED COMPONENTS Interpreter)
set(STENO_DICTS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/dicts)
set(STENO_FETCH ${CMAKE_CURRENT_SOURCE_DIR}/tools/fetch_dict.py)
# ── Dictionary selection ─────────────────────────────
# Maps the Kconfig choice onto the compiler's --dicts argument.
if(CONFIG_STENO_DICT_PLOVER)
set(STENO_DICTS_ARG "plover")
elseif(CONFIG_STENO_DICT_LAPWING)
set(STENO_DICTS_ARG "lapwing")
else()
set(STENO_DICTS_ARG "both")
endif()
# Ensure a dictionary JSON is present: download at configure time if
# missing (hard error on failure), and add a build-time stamp rule that
# re-checks the upstream hash on fresh build directories.
function(steno_fetch_dict name json_path)
if(NOT EXISTS ${json_path})
message(STATUS "Steno: downloading ${name} dictionary...")
execute_process(
COMMAND ${Python3_EXECUTABLE} ${STENO_FETCH} ${name} ${STENO_DICTS_DIR}
RESULT_VARIABLE FETCH_RESULT
)
if(NOT FETCH_RESULT EQUAL 0 OR NOT EXISTS ${json_path})
message(FATAL_ERROR
"Steno: ${name} dictionary is missing and the download failed "
"(expected ${json_path}).")
endif()
endif()
add_custom_command(
OUTPUT ${json_path}.stamp
COMMAND ${Python3_EXECUTABLE} ${STENO_FETCH} ${name} ${STENO_DICTS_DIR}
COMMAND ${CMAKE_COMMAND} -E touch ${json_path}.stamp
COMMENT "Checking ${name} dictionary for updates"
)
endfunction()
# ── v4 union split-section format ────────────────────
# One compiler run emits BOTH half blobs (docs/FORMAT_V4.md §5).
# Left (central): DISP, MEMBERSHIP, FP, CONFLICTS, VALIDX slice [0, k).
# Right (peripheral): STRDIR, STRINGS, CONFLICTS, VALIDX slice [k, n).
if(CONFIG_STENO_DICT_V4)
if(NOT CONFIG_STENO_SPLIT_DICT)
message(FATAL_ERROR
"STENO_DICT_V4 requires STENO_SPLIT_DICT: the v4 format spans both "
"keyboard halves (left = lookup structures, right = string table). "
"Enable STENO_SPLIT_DICT, or disable STENO_DICT_V4 to fall back to "
"the legacy MPHF format.")
endif()
set(STENO_V4_LEFT_BIN ${CMAKE_CURRENT_BINARY_DIR}/steno_v4_left.bin)
set(STENO_V4_RIGHT_BIN ${CMAKE_CURRENT_BINARY_DIR}/steno_v4_right.bin)
if(CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
# Central: behavior engine + local dict + BLE client for remote partition
# Central: behavior engine + left half blob + BLE client
target_sources(app PRIVATE
src/behavior_steno.c
src/output.c
src/formatter.c
src/undo.c
src/dict_v4.c
src/split_dict.c
src/split_cache.c
src/dict_embed.S
)
set(STENO_DICT_BIN ${STENO_V4_LEFT_BIN})
else()
# Peripheral: right half blob + GATT server
target_sources(app PRIVATE
src/dict_v4.c
src/split_dict.c
src/split_cache.c
src/dict_embed.S
)
set(STENO_DICT_BIN ${STENO_V4_RIGHT_BIN})
endif()
# The union format is always compiled from both source dicts;
# --dicts controls which of them are included (dicts_mask).
set(STENO_PLOVER_JSON ${STENO_DICTS_DIR}/plover-main.json)
set(STENO_LAPWING_JSON ${STENO_DICTS_DIR}/lapwing.json)
steno_fetch_dict(plover ${STENO_PLOVER_JSON})
steno_fetch_dict(lapwing ${STENO_LAPWING_JSON})
# Single run produces both blobs. The compiler FAILS the build if
# either half exceeds its budget — entries are never trimmed.
add_custom_command(
OUTPUT ${STENO_V4_LEFT_BIN} ${STENO_V4_RIGHT_BIN}
COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_v4.py
--plover ${STENO_PLOVER_JSON} --lapwing ${STENO_LAPWING_JSON}
--dicts ${STENO_DICTS_ARG}
--out-dir ${CMAKE_CURRENT_BINARY_DIR}
--left-size ${CONFIG_STENO_DICT_LEFT_MAX_SIZE}
--right-size ${CONFIG_STENO_DICT_RIGHT_MAX_SIZE}
DEPENDS
${STENO_PLOVER_JSON} ${STENO_LAPWING_JSON}
${STENO_PLOVER_JSON}.stamp ${STENO_LAPWING_JSON}.stamp
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_v4.py
COMMENT "Compiling steno v4 union dictionary (left + right half blobs)"
)
add_custom_target(steno_dict_gen DEPENDS ${STENO_DICT_BIN})
add_dependencies(app steno_dict_gen)
# ── Legacy MPHF format (STENO_DICT_V4=n fallback) ────
else()
if(CONFIG_STENO_DICT_BOTH)
message(FATAL_ERROR
"STENO_DICT_BOTH requires STENO_DICT_V4: the legacy MPHF format "
"holds a single dictionary. Select STENO_DICT_PLOVER or "
"STENO_DICT_LAPWING, or re-enable STENO_DICT_V4.")
endif()
if(CONFIG_STENO_DICT_PLOVER)
set(STENO_DICT_NAME "plover")
set(STENO_DICT_SRC ${STENO_DICTS_DIR}/plover-main.json)
else()
set(STENO_DICT_NAME "lapwing")
set(STENO_DICT_SRC ${STENO_DICTS_DIR}/lapwing.json)
endif()
if(CONFIG_STENO_SPLIT_DICT)
if(CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
# Central: behavior engine + local dict + BLE client
target_sources(app PRIVATE
src/behavior_steno.c
src/output.c
@ -35,81 +156,31 @@ if(CONFIG_STENO_SPLIT_DICT)
src/split_cache.c
)
endif()
# ── Non-split mode ───────────────────────────────────
# Everything on one board (or central-only without split)
else()
if(NOT CONFIG_ZMK_SPLIT OR CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
elseif(NOT CONFIG_ZMK_SPLIT OR CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
target_sources(app PRIVATE
src/behavior_steno.c
src/output.c
src/formatter.c
src/undo.c
src/dict_embed.S
src/dict_mphf.c
)
if(CONFIG_STENO_DICT_MPHF)
target_sources(app PRIVATE src/dict_mphf.c)
else()
target_sources(app PRIVATE src/trie.c)
endif()
endif()
endif()
# ── Dictionary compilation ───────────────────────────
# Both halves need dict embed in split mode; non-split same as before
# Dict embed needed on: both halves in split mode, else central /
# non-split only.
set(STENO_NEED_DICT_EMBED FALSE)
if(CONFIG_STENO_SPLIT_DICT)
# Both central and peripheral need their own partition
set(STENO_NEED_DICT_EMBED TRUE)
elseif(NOT CONFIG_ZMK_SPLIT OR CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
set(STENO_NEED_DICT_EMBED TRUE)
endif()
if(STENO_NEED_DICT_EMBED)
find_package(Python3 REQUIRED COMPONENTS Interpreter)
set(STENO_DICT_BIN ${CMAKE_CURRENT_BINARY_DIR}/steno_dict.bin)
set(STENO_DICTS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/dicts)
set(STENO_FETCH ${CMAKE_CURRENT_SOURCE_DIR}/tools/fetch_dict.py)
if(CONFIG_STENO_DICT_PLOVER)
set(STENO_DICT_NAME "plover")
set(STENO_DICT_SRC ${STENO_DICTS_DIR}/plover-main.json)
elseif(CONFIG_STENO_DICT_LAPWING)
set(STENO_DICT_NAME "lapwing")
set(STENO_DICT_SRC ${STENO_DICTS_DIR}/lapwing.json)
else()
set(STENO_DICT_SRC ${STENO_DICTS_DIR}/test.json)
endif()
# Auto-download dict if needed (Plover/Lapwing only)
if(DEFINED STENO_DICT_NAME AND NOT EXISTS ${STENO_DICT_SRC})
message(STATUS "Steno: downloading ${STENO_DICT_NAME} dictionary...")
execute_process(
COMMAND ${Python3_EXECUTABLE} ${STENO_FETCH}
${STENO_DICT_NAME} ${STENO_DICTS_DIR}
RESULT_VARIABLE FETCH_RESULT
)
if(NOT FETCH_RESULT EQUAL 0)
message(WARNING "Steno: dict download failed. Build may fail.")
endif()
endif()
# Compile dict binary
if(EXISTS ${STENO_DICT_SRC})
# Fetch at build time if hash changed
if(DEFINED STENO_DICT_NAME)
add_custom_command(
OUTPUT ${STENO_DICT_SRC}.stamp
COMMAND ${Python3_EXECUTABLE} ${STENO_FETCH}
${STENO_DICT_NAME} ${STENO_DICTS_DIR}
COMMAND ${CMAKE_COMMAND} -E touch ${STENO_DICT_SRC}.stamp
COMMENT "Checking ${STENO_DICT_NAME} dictionary for updates"
)
add_custom_target(steno_dict_fetch DEPENDS ${STENO_DICT_SRC}.stamp)
endif()
steno_fetch_dict(${STENO_DICT_NAME} ${STENO_DICT_SRC})
if(CONFIG_STENO_SPLIT_DICT)
# Split mode: compile the appropriate partition
if(CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
set(STENO_SPLIT_PART "left")
set(STENO_SPLIT_SIZE ${CONFIG_STENO_DICT_LEFT_MAX_SIZE})
@ -128,43 +199,39 @@ if(STENO_NEED_DICT_EMBED)
--right-size ${CONFIG_STENO_DICT_RIGHT_MAX_SIZE}
--max-size ${STENO_SPLIT_SIZE}
--block-size 2048
DEPENDS ${STENO_DICT_SRC}
DEPENDS ${STENO_DICT_SRC} ${STENO_DICT_SRC}.stamp
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_mphf.py
COMMENT "Compiling steno dictionary (${STENO_SPLIT_PART} partition)"
)
elseif(CONFIG_STENO_DICT_MPHF)
else()
add_custom_command(
OUTPUT ${STENO_DICT_BIN}
COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_mphf.py
${STENO_DICT_SRC} ${STENO_DICT_BIN}
--max-size ${CONFIG_STENO_DICT_MAX_SIZE}
DEPENDS ${STENO_DICT_SRC}
DEPENDS ${STENO_DICT_SRC} ${STENO_DICT_SRC}.stamp
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_mphf.py
COMMENT "Compiling steno dictionary (MPHF)"
)
else()
add_custom_command(
OUTPUT ${STENO_DICT_BIN}
COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_simple.py
${STENO_DICT_SRC} -o ${STENO_DICT_BIN}
DEPENDS ${STENO_DICT_SRC}
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_simple.py
COMMENT "Compiling steno dictionary (simple)"
)
endif()
add_custom_target(steno_dict_gen DEPENDS ${STENO_DICT_BIN})
if(TARGET steno_dict_fetch)
add_dependencies(steno_dict_gen steno_dict_fetch)
endif()
add_dependencies(app steno_dict_gen)
endif()
endif() # CONFIG_STENO_DICT_V4
# ── Dict embed plumbing ──────────────────────────────
# dict_embed.S .incbin's the blob named by the generated header;
# OBJECT_DEPENDS makes it re-assemble when the blob changes.
if(STENO_DICT_BIN)
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/steno_dict_path.h
"#define STENO_DICT_BIN_PATH \"${STENO_DICT_BIN}\"\n")
target_include_directories(app PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
endif()
set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/src/dict_embed.S
TARGET_DIRECTORY app
PROPERTIES OBJECT_DEPENDS ${STENO_DICT_BIN})
endif()
endif() # CONFIG_STENO_ENGINE

74
Kconfig
View file

@ -11,29 +11,43 @@ if STENO_ENGINE
choice STENO_DICT
prompt "Steno dictionary"
default STENO_DICT_PLOVER
default STENO_DICT_BOTH
config STENO_DICT_BOTH
bool "Plover + Lapwing"
help
Compile both full dictionaries into the v4 union format.
The active dictionary is selected at runtime.
config STENO_DICT_PLOVER
bool "Plover main dictionary (MPHF compressed)"
bool "Plover main dictionary"
help
Use Plover main.json via MPHF compression.
Compile only the Plover main dictionary.
config STENO_DICT_LAPWING
bool "Lapwing dictionary (MPHF compressed)"
config STENO_DICT_TEST
bool "Test dictionary (46 entries, simple format)"
bool "Lapwing dictionary"
help
Small built-in test dictionary for development.
Compile only the Lapwing dictionary.
endchoice
config STENO_DICT_MPHF
bool
default y if STENO_DICT_PLOVER || STENO_DICT_LAPWING
config STENO_DICT_V4
bool "v4 union split-section dictionary format"
default y
select ZLIB
help
Use MPHF (minimal perfect hash) dictionary format.
Compile the dictionary into the v4 union split-section format
(docs/FORMAT_V4.md): two half blobs, left (central) holds the
lookup structures, right (peripheral) holds the string table.
Requires STENO_SPLIT_DICT. Disable to fall back to the legacy
single-dictionary MPHF format.
config STENO_DICT_MPHF
bool
default y if !STENO_DICT_V4
select ZLIB
help
Legacy MPHF (minimal perfect hash) dictionary format.
Selects ZLIB for block-compressed string table decompression.
config STENO_CUSTOM_KEYMAP
@ -75,39 +89,43 @@ config STENO_MULTI_STROKE_TIMEOUT_MS
config STENO_DICT_MAX_SIZE
int "Max dictionary binary size (bytes)"
depends on !STENO_DICT_V4
default 473088
help
Max compiled dict size. 473088 = 462KB.
Build fails if dict exceeds this — no trimming.
Max compiled dict size for legacy MPHF single-blob builds.
473088 = 462KB. Build FAILS if the dict exceeds this —
entries are never trimmed.
menuconfig STENO_SPLIT_DICT
bool "Split dictionary across both halves"
default n
select STENO_DICT_MPHF
help
Partition the steno dictionary across both keyboard halves.
Left (central) gets highest-importance entries for zero-latency
local lookup. Right (peripheral) gets remaining entries, queried
over BLE on local miss. Both halves have their own MPHF dict.
Store the steno dictionary across both keyboard halves.
v4 format: left (central) holds the lookup structures
(displacements, membership, fingerprints, conflicts, part of
the value index); right (peripheral) holds the string table,
queried over BLE. Legacy MPHF format: entries partitioned by
importance, right half queried over BLE on local miss.
Build FAILS if either half exceeds its budget — entries are
never trimmed.
if STENO_SPLIT_DICT
config STENO_DICT_LEFT_MAX_SIZE
int "Left half dict budget (bytes)"
default 153600
default 440320
help
Flash budget for left (central) partition. Default 150KB.
Central has more code overhead (behavior, formatter, output,
undo, BLE client) so gets a smaller dict partition.
Highest-importance entries fill this first.
Flash budget for the left (central) half blob. Default
440320 = 430KB. Build FAILS if the compiled blob exceeds
this — entries are never trimmed.
config STENO_DICT_RIGHT_MAX_SIZE
int "Right half dict budget (bytes)"
default 512000
default 527360
help
Flash budget for right (peripheral) partition. Default 500KB.
Peripheral has less code overhead so gets the bulk of the dict.
Remaining entries after left partition is filled.
Flash budget for the right (peripheral) half blob. Default
527360 = 515KB. Build FAILS if the compiled blob exceeds
this — entries are never trimmed.
config STENO_SPLIT_CACHE_SIZE
int "LRU cache entries on central side"

402
src/behavior_steno.c
View file

@ -12,85 +12,115 @@
#include <drivers/behavior.h>
#include <zmk/behavior.h>
#include <string.h>
#include "output.h"
#include "undo.h"
#include "formatter.h"
#if IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
#include "split_dict.h"
#include "dict_mphf.h"
#elif IS_ENABLED(CONFIG_STENO_DICT_MPHF)
#include "dict_mphf.h"
#else
#include "trie.h"
#endif
LOG_MODULE_DECLARE(zmk, CONFIG_ZMK_LOG_LEVEL);
extern const uint8_t _steno_dict_start[];
extern const uint8_t _steno_dict_end[];
/*
* Dictionary dispatcher — implemented by the active dict backend
* (dict_v4 / split). Exact-match lookup of `count` strokes: returns
* the translation length (>= 0, `out` NUL-terminated) on a hit, or a
* negative value on miss/error. steno_dict_max_strokes() reports the
* longest entry in the loaded dictionary (0 if none loaded).
*/
extern int steno_dict_lookup(const uint32_t *strokes, uint8_t count,
char *out, size_t out_size);
extern uint8_t steno_dict_max_strokes(void);
#if IS_ENABLED(CONFIG_STENO_DICT_MPHF) || IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
static struct dict_mphf mphf_dict;
#endif
/*
* Runtime dictionary toggle (0 = plover, 1 = lapwing), implemented by
* the dict backend.
*
* TODO: wire steno_dict_set_active() to a dedicated chord or behavior
* parameter once a binding is assigned; no new DTS binding yet.
*/
extern void steno_dict_set_active(uint8_t dict_id);
#define STENO_MAX_MULTI 8
#define STENO_MULTI_TIMEOUT_MS CONFIG_STENO_MULTI_STROKE_TIMEOUT_MS
#define STENO_BIT_STAR 9
struct steno_state {
uint32_t current_chord;
uint8_t keys_held;
uint32_t pending_strokes[STENO_MAX_MULTI];
uint8_t stroke_count;
struct k_work_delayable multi_timeout;
};
static struct steno_state state;
static struct steno_fmt_state fmt_state;
static struct stroke_history undo_history;
static bool dict_ready;
static void flush_strokes(void);
static void multi_timeout_handler(struct k_work *work);
/* ── formatter state snapshots (packed into history fmt_flags) ─── */
static const char *do_lookup(const uint32_t *strokes, uint8_t count)
static uint8_t fmt_pack(const struct steno_fmt_state *s)
{
#if IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
/* Try local partition first (zero latency) */
const char *local = dict_mphf_lookup(&mphf_dict, strokes, count);
if (local) {
return local;
}
/* Miss → query remote partition over BLE */
static char split_buf[128];
int ret = split_dict_lookup(strokes, count, split_buf, sizeof(split_buf));
return (ret > 0) ? split_buf : NULL;
#elif IS_ENABLED(CONFIG_STENO_DICT_MPHF)
return dict_mphf_lookup(&mphf_dict, strokes, count);
#else
return steno_trie_lookup(strokes, count);
#endif
return (uint8_t)((s->space_pending ? 0x01 : 0) |
(s->cap_next ? 0x02 : 0) |
(s->upper_next ? 0x04 : 0) |
(s->lower_next ? 0x08 : 0) |
(s->suppress_space ? 0x10 : 0) |
(s->glue ? 0x20 : 0) |
(((uint8_t)s->mode & 0x03) << 6));
}
static bool do_has_prefix(const uint32_t *strokes, uint8_t count)
static void fmt_unpack(uint8_t flags, struct steno_fmt_state *s)
{
#if IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
/* Check local prefix table first */
if (count == 1 && dict_mphf_has_prefix(&mphf_dict, strokes[0])) {
return true;
s->space_pending = (flags & 0x01) != 0;
s->cap_next = (flags & 0x02) != 0;
s->upper_next = (flags & 0x04) != 0;
s->lower_next = (flags & 0x08) != 0;
s->suppress_space = (flags & 0x10) != 0;
s->glue = (flags & 0x20) != 0;
s->mode = (enum steno_fmt_mode)((flags >> 6) & 0x03);
}
/* Fall through to remote */
return split_dict_has_prefix(strokes, count);
#elif IS_ENABLED(CONFIG_STENO_DICT_MPHF)
return (count == 1) ? dict_mphf_has_prefix(&mphf_dict, strokes[0]) : false;
#else
return steno_trie_has_prefix(strokes, count);
#endif
/* ── raw steno rendering (total-miss fallback) ──────────────────── */
/* '#' + 22 keys + '-' + NUL */
#define STENO_RAW_MAX 26
/* Bit layout per parse_stroke (tools/compile_mphf.py, FORMAT_V4.md) */
static const char steno_key_char[22] = {
'S', 'T', 'K', 'P', 'W', 'H', 'R', /* bits 0-6: left */
'A', 'O', '*', 'E', 'U', /* bits 7-11: mid */
'F', 'R', 'P', 'B', 'L', 'G', 'T', 'S', 'D', 'Z', /* bits 12-21: right */
};
#define STENO_MASK_MIDDLE 0x00000F80U /* A O * E U — implicit hyphen */
#define STENO_MASK_NUM 0x00400000U /* # */
#define STENO_FIRST_RIGHT 12
static void stroke_to_steno(uint32_t stroke, char out[STENO_RAW_MAX])
{
uint8_t len = 0;
bool implicit = (stroke & STENO_MASK_MIDDLE) != 0;
if (stroke & STENO_MASK_NUM) {
out[len++] = '#';
}
for (uint8_t bit = 0; bit < 22; bit++) {
if (!(stroke & (1U << bit))) {
continue;
}
if (bit >= STENO_FIRST_RIGHT && !implicit) {
out[len++] = '-';
implicit = true;
}
out[len++] = steno_key_char[bit];
}
out[len] = '\0';
}
/* ── emission ───────────────────────────────────────────────────── */
static void emit_formatted(const char *translation,
const uint32_t *strokes, uint8_t stroke_count)
{
/* Snapshot formatter state BEFORE this translation mutates it, so
* undo/retrace can restore it. */
uint8_t snap = fmt_pack(&fmt_state);
struct steno_fmt_result result;
steno_fmt_process(&fmt_state, translation, &result);
if (result.backspaces > 0) {
@ -100,98 +130,180 @@ static void emit_formatted(const char *translation,
steno_output_send(result.text, result.len);
}
if (!result.is_command_only) {
uint16_t out_chars = (uint16_t)result.len + result.backspaces;
if (out_chars > 255) {
out_chars = 255;
}
steno_undo_push(&undo_history, strokes, stroke_count,
result.len + result.backspaces, 0, 0);
(uint8_t)out_chars, 0, snap);
}
}
static void emit_raw_stroke(uint32_t stroke)
{
char raw[STENO_RAW_MAX];
stroke_to_steno(stroke, raw);
emit_formatted(raw, &stroke, 1);
}
/*
* Greedy longest-match over a contiguous stroke span. Used to
* re-translate strokes orphaned by a retrace. No further retrace:
* matches are confined to the span.
*/
static void translate_span(const uint32_t *strokes, uint8_t count,
uint8_t max_win)
{
uint8_t pos = 0;
while (pos < count) {
uint8_t span = count - pos;
char text[STENO_FMT_MAX_OUTPUT];
uint8_t match_len = 0;
if (span > max_win) {
span = max_win;
}
for (uint8_t l = span; l >= 1; l--) {
if (steno_dict_lookup(&strokes[pos], l, text, sizeof(text)) >= 0) {
match_len = l;
break;
}
}
if (match_len == 0) {
emit_raw_stroke(strokes[pos]);
pos++;
} else {
emit_formatted(text, &strokes[pos], match_len);
pos += match_len;
}
}
}
/* ── sliding longest-match translation with retrace ─────────────── */
static void translate_stroke(uint32_t stroke)
{
uint8_t max_win = steno_dict_max_strokes();
if (max_win < 1) {
max_win = 1;
} else if (max_win > STENO_MAX_MULTI_STROKE) {
max_win = STENO_MAX_MULTI_STROKE;
}
/*
* Build the lookup window: win[max_win - 1] is the new stroke,
* earlier strokes (pulled from translation history, newest first)
* fill leftward. avail = previous strokes actually gathered.
*/
uint32_t win[STENO_MAX_MULTI_STROKE];
uint8_t avail = 0;
win[max_win - 1] = stroke;
for (uint16_t i = 0; avail < max_win - 1; i++) {
const struct stroke_history_entry *e =
steno_undo_peek_at(&undo_history, i);
if (!e) {
break;
}
for (uint8_t k = e->stroke_count; k > 0 && avail < max_win - 1; k--) {
avail++;
win[max_win - 1 - avail] = e->strokes[k - 1];
}
}
/* Longest match wins: L = avail + 1 down to 1 */
char text[STENO_FMT_MAX_OUTPUT];
uint8_t match_len = 0;
for (uint8_t l = avail + 1; l >= 1; l--) {
if (steno_dict_lookup(&win[max_win - l], l, text, sizeof(text)) >= 0) {
match_len = l;
break;
}
}
if (match_len == 0) {
/* Total miss → raw steno chars for this stroke */
emit_raw_stroke(stroke);
return;
}
if (match_len == 1) {
emit_formatted(text, &stroke, 1);
return;
}
/*
* Retrace: the match swallows strokes already emitted by earlier
* translations. Pop those, erase their output, restore the
* formatter state that preceded the oldest popped entry, then
* re-emit.
*/
uint8_t needed = match_len - 1;
uint8_t covered = 0;
uint16_t erase = 0;
uint8_t snap = fmt_pack(&fmt_state);
uint32_t orphan[STENO_MAX_MULTI_STROKE];
uint8_t orphan_count = 0;
while (covered < needed) {
struct stroke_history_entry *e = steno_undo_pop(&undo_history);
if (!e) {
/* Cannot happen: avail came from the same entries */
break;
}
covered += e->stroke_count;
erase += e->output_len;
snap = e->fmt_flags;
if (covered > needed) {
/*
* Oldest popped entry straddles the window boundary: its
* leading strokes fall outside the new match and must be
* re-translated. Copy before pushes reuse the ring slot.
*/
orphan_count = covered - needed;
memcpy(orphan, e->strokes, orphan_count * sizeof(uint32_t));
}
}
if (erase > 0) {
steno_output_backspace(erase);
}
fmt_unpack(snap, &fmt_state);
if (orphan_count > 0) {
translate_span(orphan, orphan_count, max_win);
}
emit_formatted(text, &win[max_win - match_len], match_len);
}
/* ── chord assembly ─────────────────────────────────────────────── */
static void process_chord(void)
{
if (state.current_chord == 0) {
uint32_t stroke = state.current_chord;
state.current_chord = 0;
if (stroke == 0) {
return;
}
/* Star-only stroke (bit 9) → undo */
if (state.current_chord == (1U << 9)) {
state.current_chord = 0;
/* Star-only stroke → undo last translation */
if (stroke == (1U << STENO_BIT_STAR)) {
struct stroke_history_entry *ue = steno_undo_pop(&undo_history);
if (ue) {
if (ue->output_len > 0) {
steno_output_backspace(ue->output_len);
}
return;
}
k_work_cancel_delayable(&state.multi_timeout);
if (state.stroke_count < STENO_MAX_MULTI) {
state.pending_strokes[state.stroke_count] = state.current_chord;
state.stroke_count++;
} else {
flush_strokes();
state.pending_strokes[0] = state.current_chord;
state.stroke_count = 1;
}
state.current_chord = 0;
if (!dict_ready) {
flush_strokes();
return;
}
const char *translation = do_lookup(state.pending_strokes, state.stroke_count);
if (translation) {
emit_formatted(translation, state.pending_strokes, state.stroke_count);
state.stroke_count = 0;
return;
}
if (do_has_prefix(state.pending_strokes, state.stroke_count)) {
k_work_schedule(&state.multi_timeout,
K_MSEC(STENO_MULTI_TIMEOUT_MS));
return;
}
if (state.stroke_count > 1) {
uint32_t last = state.pending_strokes[state.stroke_count - 1];
state.stroke_count--;
const char *partial = do_lookup(state.pending_strokes, state.stroke_count);
if (partial) {
emit_formatted(partial, state.pending_strokes, state.stroke_count);
}
state.pending_strokes[0] = last;
state.stroke_count = 1;
const char *rest = do_lookup(&last, 1);
if (rest) {
emit_formatted(rest, &last, 1);
state.stroke_count = 0;
fmt_unpack(ue->fmt_flags, &fmt_state);
}
return;
}
flush_strokes();
}
static void flush_strokes(void)
{
LOG_DBG("Flushing %u untranslated strokes", state.stroke_count);
state.stroke_count = 0;
}
static void multi_timeout_handler(struct k_work *work)
{
ARG_UNUSED(work);
if (state.stroke_count == 0) {
return;
}
const char *translation = do_lookup(state.pending_strokes, state.stroke_count);
if (translation) {
emit_formatted(translation, state.pending_strokes, state.stroke_count);
}
state.stroke_count = 0;
translate_stroke(stroke);
}
static int on_steno_binding_pressed(struct zmk_behavior_binding *binding,
@ -228,49 +340,9 @@ static int behavior_steno_init(const struct device *dev)
state.current_chord = 0;
state.keys_held = 0;
state.stroke_count = 0;
steno_fmt_init(&fmt_state);
steno_undo_init(&undo_history);
k_work_init_delayable(&state.multi_timeout, multi_timeout_handler);
#if IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
/* Init local partition dict */
{
size_t dict_size = _steno_dict_end - _steno_dict_start;
if (dict_size > 4) {
int ret = dict_mphf_init(&mphf_dict, _steno_dict_start, dict_size);
if (ret == 0) {
LOG_INF("Local partition loaded (%u bytes)", (unsigned)dict_size);
} else {
LOG_ERR("Local partition init failed: %d", ret);
}
} else {
LOG_WRN("No local partition embedded");
}
}
/* Init BLE client for remote partition */
split_dict_init();
dict_ready = true;
#else
size_t dict_size = _steno_dict_end - _steno_dict_start;
if (dict_size > 4) {
int ret;
#if IS_ENABLED(CONFIG_STENO_DICT_MPHF)
ret = dict_mphf_init(&mphf_dict, _steno_dict_start, dict_size);
#else
ret = steno_trie_init(_steno_dict_start, dict_size);
#endif
if (ret == 0) {
dict_ready = true;
LOG_INF("Steno dict loaded (%u bytes)", (unsigned)dict_size);
} else {
LOG_ERR("Steno dict init failed: %d", ret);
}
} else {
LOG_WRN("No steno dict embedded");
}
#endif
LOG_INF("Steno engine initialized");
return 0;

16
src/dict_embed.S
View file

@ -1,14 +1,30 @@
/*
* Copyright (c) 2024 Afiq Zudin Hadi
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*
* Embeds the compiled dictionary blob selected by the build system.
* steno_dict_path.h is generated by CMake and names the blob for this
* half (v4: steno_v4_left.bin / steno_v4_right.bin; legacy MPHF:
* steno_dict.bin). Both symbol spellings are exported: legacy code
* uses _steno_dict_start/_steno_dict_end, dict_v4.c uses
* steno_dict_start/steno_dict_end.
*/
#include "steno_dict_path.h"
.section .rodata.steno_dict, "a", %progbits
.global _steno_dict_start
.global _steno_dict_end
.global steno_dict_start
.global steno_dict_end
.balign 4
_steno_dict_start:
steno_dict_start:
#ifdef STENO_DICT_BIN_PATH
.incbin STENO_DICT_BIN_PATH
#else
.byte 0x00, 0x00, 0x00, 0x00
#endif
_steno_dict_end:
steno_dict_end:

839
src/dict_v4.c Normal file
View file

@ -0,0 +1,839 @@
/*
* Copyright (c) 2024 zmk-steno-engine contributors
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*
* Licensed under the PolyForm Noncommercial License 1.0.0;
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* https://polyformproject.org/licenses/noncommercial/1.0.0
*/
/*
* Dictionary binary format v4 decoder — implementation.
*
* All data is read directly from flash. No heap allocation on the
* Zephyr path; string blocks are inflated on demand into a static
* two-slot LRU cache (16 KB per slot, the compile-time block bound).
*/
#include "dict_v4.h"
#include <errno.h>
#include <stdbool.h>
#include <string.h>
#define DICT_V4_ARRAY_LEN(a) (sizeof(a) / sizeof((a)[0]))
/* ─── Raw-deflate block inflate (wbits = -15) ─── */
#ifndef __ZEPHYR__
/* Use zlib on host for native tests */
#ifdef HAS_ZLIB
#include <zlib.h>
static int block_inflate_raw(const uint8_t *src, size_t src_len,
uint8_t *dst, size_t dst_cap, size_t *dst_len)
{
z_stream strm;
memset(&strm, 0, sizeof(strm));
if (inflateInit2(&strm, -15) != Z_OK) {
return -1;
}
strm.next_in = (Bytef *)(uintptr_t)src;
strm.avail_in = (uInt)src_len;
strm.next_out = dst;
strm.avail_out = (uInt)dst_cap;
int ret = inflate(&strm, Z_FINISH);
inflateEnd(&strm);
if (ret == Z_STREAM_END) {
*dst_len = dst_cap - strm.avail_out;
return 0;
}
return -1;
}
#else
static int block_inflate_raw(const uint8_t *src, size_t src_len,
uint8_t *dst, size_t dst_cap, size_t *dst_len)
{
/* No zlib on host — string sections unavailable */
(void)src; (void)src_len; (void)dst; (void)dst_cap; (void)dst_len;
return -1;
}
#endif /* HAS_ZLIB */
#else /* __ZEPHYR__ */
#include <zephyr/sys/util.h>
#if __has_include(<zephyr/lib/zlib/zlib.h>)
#include <zephyr/lib/zlib/zlib.h>
#elif __has_include(<zlib.h>)
#include <zlib.h>
#endif
static int block_inflate_raw(const uint8_t *src, size_t src_len,
uint8_t *dst, size_t dst_cap, size_t *dst_len)
{
#if defined(CONFIG_ZLIB)
z_stream strm;
memset(&strm, 0, sizeof(strm));
if (inflateInit2(&strm, -15) != Z_OK) {
return -1;
}
strm.next_in = (Bytef *)(uintptr_t)src;
strm.avail_in = (uInt)src_len;
strm.next_out = dst;
strm.avail_out = (uInt)dst_cap;
int ret = inflate(&strm, Z_FINISH);
inflateEnd(&strm);
if (ret == Z_STREAM_END) {
*dst_len = dst_cap - strm.avail_out;
return 0;
}
return -1;
#else
/* No zlib available — the v4 string path cannot run on this half */
(void)src; (void)src_len; (void)dst; (void)dst_cap; (void)dst_len;
return -1;
#endif
}
#endif /* __ZEPHYR__ */
/* ─── Byte helpers (blob fields may be unaligned) ─── */
static uint32_t get_le32(const uint8_t *p)
{
return (uint32_t)p[0]
| ((uint32_t)p[1] << 8)
| ((uint32_t)p[2] << 16)
| ((uint32_t)p[3] << 24);
}
/* ─── FNV-1a 32-bit hash (identical to v2) ─── */
static uint32_t fnv1a_32(const uint8_t *data, size_t len)
{
uint32_t h = 0x811c9dc5u;
for (size_t i = 0; i < len; i++) {
h ^= data[i];
h *= 0x01000193u;
}
return h;
}
static uint32_t hash_key(const uint8_t *key, size_t key_len, uint32_t seed)
{
uint32_t h = 0x811c9dc5u;
uint8_t seed_bytes[4];
seed_bytes[0] = (uint8_t)(seed);
seed_bytes[1] = (uint8_t)(seed >> 8);
seed_bytes[2] = (uint8_t)(seed >> 16);
seed_bytes[3] = (uint8_t)(seed >> 24);
for (int i = 0; i < 4; i++) {
h ^= seed_bytes[i];
h *= 0x01000193u;
}
for (size_t i = 0; i < key_len; i++) {
h ^= key[i];
h *= 0x01000193u;
}
return h;
}
/* ─── LSB-first bit-packed field reading (MEMBERSHIP/VALIDX/FP) ─── */
static uint32_t read_bits_lsb(const uint8_t *data, uint32_t bit_pos, uint8_t n_bits)
{
uint32_t value = 0;
for (uint8_t i = 0; i < n_bits; i++) {
uint32_t byte_idx = (bit_pos + i) / 8;
uint8_t bit_idx = (bit_pos + i) % 8;
if (data[byte_idx] & (1u << bit_idx)) {
value |= (1u << i);
}
}
return value;
}
/* ─── MSB-first bit reading (DISP Huffman stream only) ─── */
static inline uint32_t read_bit_msb(const uint8_t *data, uint32_t bit_pos)
{
return (data[bit_pos >> 3] >> (7 - (bit_pos & 7))) & 1u;
}
/* ─── Canonical Huffman (DISP displacement classes) ─── */
static int huff_build(struct dict_v4 *d)
{
const uint8_t *code_len = d->disp_code_len;
uint8_t max_len = 0;
memset(d->huff_count, 0, sizeof(d->huff_count));
for (int c = 0; c < DICT_V4_DISP_CLASSES; c++) {
uint8_t l = code_len[c];
if (l == 0) {
continue;
}
if (l > DICT_V4_DISP_CLASSES) {
return -EBADMSG;
}
d->huff_count[l]++;
if (l > max_len) {
max_len = l;
}
}
if (max_len == 0) {
return -EBADMSG;
}
/* Symbol list in canonical (code_len, class) order */
uint8_t idx = 0;
for (uint8_t l = 1; l <= max_len; l++) {
d->huff_sym_base[l] = idx;
for (uint8_t c = 0; c < DICT_V4_DISP_CLASSES; c++) {
if (code_len[c] == l) {
d->huff_symbols[idx++] = c;
}
}
}
/* First canonical code per length; codes assigned numerically
* increasing, MSB-first prefix codes. */
uint32_t code = 0;
for (uint8_t l = 1; l <= max_len; l++) {
d->huff_first_code[l] = code;
code += d->huff_count[l];
if (code > (1u << l)) {
return -EBADMSG; /* over-subscribed code space */
}
code <<= 1;
}
d->huff_max_len = max_len;
return 0;
}
/* Decode one displacement value at *bit_pos, advancing it. */
static int huff_decode_one(const struct dict_v4 *d, uint32_t *bit_pos, uint32_t *v_out)
{
uint32_t pos = *bit_pos;
uint32_t code = 0;
uint8_t sym = 0;
uint8_t found = 0;
for (uint8_t len = 1; len <= d->huff_max_len; len++) {
if (pos >= d->disp_stream_bits) {
return -EBADMSG;
}
code = (code << 1) | read_bit_msb(d->disp_stream, pos++);
if (d->huff_count[len] != 0 &&
code >= d->huff_first_code[len] &&
code - d->huff_first_code[len] < d->huff_count[len]) {
sym = d->huff_symbols[d->huff_sym_base[len] +
(code - d->huff_first_code[len])];
found = 1;
break;
}
}
if (!found) {
return -EBADMSG;
}
uint32_t v;
if (sym <= 1) {
v = sym;
} else {
uint32_t extra = 0;
for (uint8_t i = 0; i < sym - 1; i++) {
if (pos >= d->disp_stream_bits) {
return -EBADMSG;
}
extra = (extra << 1) | read_bit_msb(d->disp_stream, pos++);
}
v = (1u << (sym - 1)) | extra;
}
*bit_pos = pos;
*v_out = v;
return 0;
}
/* Random access: start at skip[b >> 8], decode (b & 255) values, next
* value is bucket b's displacement. */
static int disp_decode(const struct dict_v4 *d, uint32_t bucket, uint32_t *v_out)
{
uint32_t skip_idx = bucket >> 8;
if (skip_idx >= d->disp_skip_count) {
return -EBADMSG;
}
uint32_t bit_pos = get_le32(d->disp_skip + skip_idx * 4);
uint32_t v = 0;
for (uint32_t i = 0; i <= (bucket & 255u); i++) {
int ret = huff_decode_one(d, &bit_pos, &v);
if (ret != 0) {
return ret;
}
}
*v_out = v;
return 0;
}
/* ─── CONFLICTS: binary search on sorted 5-byte records ─── */
static int conflict_lookup(const struct dict_v4 *d, uint32_t slot, uint32_t *string_id)
{
uint32_t lo = 0;
uint32_t hi = d->conflict_count;
while (lo < hi) {
uint32_t mid = lo + (hi - lo) / 2;
const uint8_t *rec = d->conflicts + mid * DICT_V4_CONFLICT_REC_SIZE;
uint64_t v = (uint64_t)rec[0]
| ((uint64_t)rec[1] << 8)
| ((uint64_t)rec[2] << 16)
| ((uint64_t)rec[3] << 24)
| ((uint64_t)rec[4] << 32);
uint32_t rec_slot = (uint32_t)(v & 0x3FFFFu);
if (rec_slot == slot) {
*string_id = (uint32_t)(v >> 18);
return 0;
}
if (rec_slot < slot) {
lo = mid + 1;
} else {
hi = mid;
}
}
return -ENOENT;
}
/* ─── Init ─── */
static int init_disp(struct dict_v4 *d, const uint8_t *sec, uint32_t sec_len)
{
uint32_t fixed = DICT_V4_DISP_CLASSES + 4;
if (sec_len < fixed) {
return -EBADMSG;
}
uint32_t skip_count = get_le32(sec + DICT_V4_DISP_CLASSES);
uint32_t expected = (d->header->bucket_count + 255u) / 256u;
if (skip_count != expected || sec_len < fixed + skip_count * 4) {
return -EBADMSG;
}
d->disp_code_len = sec;
d->disp_skip = sec + fixed;
d->disp_skip_count = skip_count;
d->disp_stream = sec + fixed + skip_count * 4;
d->disp_stream_bits = (sec_len - fixed - skip_count * 4) * 8;
return huff_build(d);
}
int dict_v4_init(struct dict_v4 *d, const void *blob, size_t len)
{
if (!d || !blob) {
return -EINVAL;
}
memset(d, 0, sizeof(*d));
if (len < sizeof(struct dict_v4_header)) {
return -EBADMSG;
}
const struct dict_v4_header *hdr = (const struct dict_v4_header *)blob;
if (hdr->magic != DICT_V4_MAGIC || hdr->version != DICT_V4_VERSION) {
return -EBADMSG;
}
const uint8_t *base = (const uint8_t *)blob;
uint32_t sc = hdr->section_count;
uint32_t dir_end = sizeof(struct dict_v4_header) +
sc * sizeof(struct dict_v4_section);
if (len < dir_end) {
return -EBADMSG;
}
d->header = hdr;
for (uint32_t i = 0; i < sc; i++) {
const struct dict_v4_section *ent = (const struct dict_v4_section *)
(base + sizeof(struct dict_v4_header) + i * sizeof(*ent));
const uint8_t *sec = base + ent->offset;
uint32_t sec_len = ent->len;
int ret;
if (ent->offset < dir_end || (ent->offset & 3u) != 0 ||
ent->offset > len || sec_len > len - ent->offset) {
return -EBADMSG;
}
switch (ent->type) {
case DICT_V4_SEC_DISP:
ret = init_disp(d, sec, sec_len);
if (ret != 0) {
return ret;
}
break;
case DICT_V4_SEC_MEMBERSHIP:
if ((uint64_t)sec_len * 8 < (uint64_t)hdr->n * 2) {
return -EBADMSG;
}
d->membership = sec;
break;
case DICT_V4_SEC_VALIDX: {
uint32_t count = (uint32_t)(((uint64_t)sec_len * 8) /
DICT_V4_VALIDX_BITS);
if (ent->param > hdr->n || count > hdr->n - ent->param) {
return -EBADMSG;
}
d->validx = sec;
d->validx_start_slot = ent->param;
d->validx_slot_count = count;
break;
}
case DICT_V4_SEC_CONFLICTS:
if (ent->param != hdr->conflict_count ||
sec_len < ent->param * DICT_V4_CONFLICT_REC_SIZE) {
return -EBADMSG;
}
d->conflicts = sec;
d->conflict_count = ent->param;
break;
case DICT_V4_SEC_FP:
if ((uint64_t)sec_len * 8 < (uint64_t)hdr->n * hdr->fp_bits) {
return -EBADMSG;
}
d->fp = sec;
break;
case DICT_V4_SEC_STRDIR: {
if (sec_len < 4) {
return -EBADMSG;
}
uint32_t block_count = get_le32(sec);
if (block_count != ent->param ||
sec_len < 4 + (uint64_t)block_count * 8) {
return -EBADMSG;
}
d->strdir_entries = sec + 4;
d->strdir_block_count = block_count;
break;
}
case DICT_V4_SEC_STRINGS:
d->strings = sec;
d->strings_len = sec_len;
break;
default:
/* Unknown section: ignore for forward compatibility */
break;
}
}
return 0;
}
/* ─── Lookup (decision path, left half) ─── */
int dict_v4_lookup(const struct dict_v4 *d, const uint32_t *strokes, uint8_t count,
uint8_t active_dict, uint32_t *slot, uint32_t *string_id)
{
if (!d || !d->header || !strokes || count == 0 || !slot || active_dict > 1) {
return -EINVAL;
}
if (!d->disp_code_len || !d->membership || !d->fp || !d->validx) {
return -ENOTSUP; /* this half lacks the decision sections */
}
const struct dict_v4_header *hdr = d->header;
if (!(hdr->dicts_mask & (1u << active_dict))) {
return DICT_V4_MISS;
}
if (count > hdr->max_entry_strokes || count > DICT_V4_MAX_KEY_STROKES) {
return DICT_V4_MISS;
}
uint8_t key_buf[DICT_V4_MAX_KEY_STROKES * 4];
size_t key_len = (size_t)count * 4;
for (uint8_t i = 0; i < count; i++) {
key_buf[i * 4 + 0] = (uint8_t)(strokes[i]);
key_buf[i * 4 + 1] = (uint8_t)(strokes[i] >> 8);
key_buf[i * 4 + 2] = (uint8_t)(strokes[i] >> 16);
key_buf[i * 4 + 3] = (uint8_t)(strokes[i] >> 24);
}
uint32_t bucket = hash_key(key_buf, key_len, 0) % hdr->bucket_count;
uint32_t v;
int ret = disp_decode(d, bucket, &v);
if (ret != 0) {
return ret;
}
uint32_t s;
if (v >= hdr->d_threshold) {
s = v - hdr->d_threshold; /* direct slot index */
if (s >= hdr->n) {
return -EBADMSG;
}
} else {
s = hash_key(key_buf, key_len, v + 1) % hdr->n;
}
uint8_t expected_fp = (uint8_t)(fnv1a_32(key_buf, key_len) &
((1u << hdr->fp_bits) - 1u));
if (read_bits_lsb(d->fp, s * hdr->fp_bits, hdr->fp_bits) != expected_fp) {
return DICT_V4_MISS;
}
uint32_t m = read_bits_lsb(d->membership, s * 2, 2);
if (!(m & (1u << active_dict))) {
return DICT_V4_MISS;
}
*slot = s;
if (s < d->validx_start_slot ||
s - d->validx_start_slot >= d->validx_slot_count) {
return DICT_V4_FOUND_REMOTE;
}
uint32_t sid = read_bits_lsb(d->validx,
(s - d->validx_start_slot) * DICT_V4_VALIDX_BITS,
DICT_V4_VALIDX_BITS);
/* Main value index holds plover's string id for both-membership
* conflicts; substitute lapwing's from the conflict table. */
if (active_dict == DICT_V4_DICT_LAPWING && m == 3 && d->conflicts) {
uint32_t conflict_sid;
if (conflict_lookup(d, s, &conflict_sid) == 0) {
sid = conflict_sid;
}
}
if (sid >= hdr->string_count) {
return -EBADMSG;
}
if (string_id) {
*string_id = sid;
}
return DICT_V4_FOUND_LOCAL;
}
/* ─── String path (right half or host test) ─── */
/* Static two-slot LRU block cache. 16 KB per slot: the compiler caps
* each front-coded block's uncompressed size at 16384 bytes. */
struct dict_v4_block_slot {
const struct dict_v4 *owner;
uint32_t block_idx;
uint32_t len;
uint32_t tick;
uint8_t buf[DICT_V4_BLOCK_BUF_SIZE];
};
static struct dict_v4_block_slot block_cache[2];
static uint32_t block_cache_tick;
static uint32_t strdir_comp_off(const struct dict_v4 *d, uint32_t block)
{
return get_le32(d->strdir_entries + block * 8);
}
static uint32_t strdir_first_sid(const struct dict_v4 *d, uint32_t block)
{
return get_le32(d->strdir_entries + block * 8 + 4);
}
static int block_get(const struct dict_v4 *d, uint32_t block,
const uint8_t **buf, uint32_t *buf_len)
{
struct dict_v4_block_slot *victim = &block_cache[0];
for (size_t i = 0; i < DICT_V4_ARRAY_LEN(block_cache); i++) {
struct dict_v4_block_slot *slot = &block_cache[i];
if (slot->owner == d && slot->block_idx == block) {
slot->tick = ++block_cache_tick;
*buf = slot->buf;
*buf_len = slot->len;
return 0;
}
if (slot->tick < victim->tick) {
victim = slot;
}
}
uint32_t comp_off = strdir_comp_off(d, block);
uint32_t comp_end = (block + 1 < d->strdir_block_count)
? strdir_comp_off(d, block + 1)
: d->strings_len;
if (comp_end < comp_off || comp_end > d->strings_len) {
return -EBADMSG;
}
size_t out_len;
victim->owner = NULL;
if (block_inflate_raw(d->strings + comp_off, comp_end - comp_off,
victim->buf, sizeof(victim->buf), &out_len) != 0) {
return -EIO;
}
victim->owner = d;
victim->block_idx = block;
victim->len = (uint32_t)out_len;
victim->tick = ++block_cache_tick;
*buf = victim->buf;
*buf_len = victim->len;
return 0;
}
int dict_v4_string_by_id_ctx(const struct dict_v4 *d, uint32_t string_id,
char *out, size_t out_size)
{
if (!d || !d->header || !out || out_size == 0) {
return -EINVAL;
}
if (!d->strdir_entries || !d->strings) {
return -ENOTSUP; /* this half lacks the string sections */
}
if (string_id >= d->header->string_count || d->strdir_block_count == 0) {
return -ENOENT;
}
/* Binary search: last block with first_string_id <= string_id */
uint32_t lo = 0;
uint32_t hi = d->strdir_block_count;
while (lo < hi) {
uint32_t mid = lo + (hi - lo) / 2;
if (strdir_first_sid(d, mid) <= string_id) {
lo = mid + 1;
} else {
hi = mid;
}
}
if (lo == 0) {
return -EBADMSG;
}
uint32_t block = lo - 1;
const uint8_t *buf;
uint32_t buf_len;
int ret = block_get(d, block, &buf, &buf_len);
if (ret != 0) {
return ret;
}
/* Walk the front-coded chain, maintaining the previous string
* in place: the shared prefix bytes are already correct. */
static uint8_t prev[DICT_V4_MAX_TEXT + 1];
uint32_t steps = string_id - strdir_first_sid(d, block);
uint32_t pos = 0;
uint32_t prev_len = 0;
for (uint32_t i = 0; i <= steps; i++) {
if (pos >= buf_len) {
return -EBADMSG;
}
uint32_t prefix = buf[pos++];
if (prefix > prev_len || (i == 0 && prefix != 0)) {
return -EBADMSG;
}
uint32_t cur_len = prefix;
while (pos < buf_len && buf[pos] != 0) {
if (cur_len >= DICT_V4_MAX_TEXT) {
return -EBADMSG;
}
prev[cur_len++] = buf[pos++];
}
if (pos >= buf_len) {
return -EBADMSG; /* missing terminator */
}
pos++; /* skip 0x00 */
prev_len = cur_len;
}
if (prev_len + 1 > out_size) {
return -ENOSPC;
}
memcpy(out, prev, prev_len);
out[prev_len] = '\0';
return (int)prev_len;
}
int dict_v4_resolve_slot_ctx(const struct dict_v4 *d, uint32_t slot, uint8_t dict,
char *out, size_t out_size)
{
if (!d || !d->header || !out || out_size == 0 || dict > 1) {
return -EINVAL;
}
if (!d->validx) {
return -ENOTSUP;
}
if (slot < d->validx_start_slot ||
slot - d->validx_start_slot >= d->validx_slot_count) {
return -ENOENT;
}
uint32_t sid = read_bits_lsb(d->validx,
(slot - d->validx_start_slot) *
DICT_V4_VALIDX_BITS,
DICT_V4_VALIDX_BITS);
if (dict == DICT_V4_DICT_LAPWING && d->conflicts) {
uint32_t conflict_sid;
if (conflict_lookup(d, slot, &conflict_sid) == 0) {
sid = conflict_sid;
}
}
if (sid >= d->header->string_count) {
return -EBADMSG;
}
return dict_v4_string_by_id_ctx(d, sid, out, out_size);
}
uint8_t dict_v4_max_strokes(const struct dict_v4 *d)
{
if (!d || !d->header) {
return 0;
}
return d->header->max_entry_strokes;
}
/* ─── Zephyr singleton wrappers (embedded blob) ─── */
#ifdef __ZEPHYR__
#ifdef CONFIG_STENO_SPLIT_DICT
/* BLE transport for the sections living on the other half (split_dict.c) */
extern int split_dict_get_string(uint32_t string_id, char *out, size_t out_size);
extern int split_dict_resolve(uint32_t slot, uint8_t dict, char *out, size_t out_size);
#endif
extern const uint8_t steno_dict_start[];
extern const uint8_t steno_dict_end[];
static struct dict_v4 dict_singleton;
static bool dict_singleton_ready;
static uint8_t active_dict_id;
static int singleton_init(void)
{
if (dict_singleton_ready) {
return 0;
}
size_t blob_len = (size_t)(steno_dict_end - steno_dict_start);
int ret = dict_v4_init(&dict_singleton, steno_dict_start, blob_len);
if (ret != 0) {
return ret;
}
active_dict_id = (dict_singleton.header->dicts_mask &
(1u << DICT_V4_DICT_PLOVER))
? DICT_V4_DICT_PLOVER : DICT_V4_DICT_LAPWING;
dict_singleton_ready = true;
return 0;
}
int dict_v4_string_by_id(uint32_t string_id, char *out, size_t out_size)
{
int ret = singleton_init();
if (ret != 0) {
return ret;
}
return dict_v4_string_by_id_ctx(&dict_singleton, string_id, out, out_size);
}
int dict_v4_resolve_slot(uint32_t slot, uint8_t dict, char *out, size_t out_size)
{
int ret = singleton_init();
if (ret != 0) {
return ret;
}
return dict_v4_resolve_slot_ctx(&dict_singleton, slot, dict, out, out_size);
}
int steno_dict_lookup(const uint32_t *strokes, uint8_t count,
char *out, size_t out_size)
{
int ret = singleton_init();
if (ret != 0) {
return ret;
}
uint32_t slot = 0;
uint32_t string_id = 0;
ret = dict_v4_lookup(&dict_singleton, strokes, count, active_dict_id,
&slot, &string_id);
if (ret == DICT_V4_FOUND_LOCAL) {
if (dict_singleton.strdir_entries && dict_singleton.strings) {
return dict_v4_string_by_id_ctx(&dict_singleton, string_id,
out, out_size);
}
#ifdef CONFIG_STENO_SPLIT_DICT
return split_dict_get_string(string_id, out, out_size);
#else
return -ENOTSUP;
#endif
}
if (ret == DICT_V4_FOUND_REMOTE) {
#ifdef CONFIG_STENO_SPLIT_DICT
return split_dict_resolve(slot, active_dict_id, out, out_size);
#else
return -ENOTSUP;
#endif
}
if (ret == DICT_V4_MISS) {
return -ENOENT;
}
return ret;
}
uint8_t steno_dict_max_strokes(void)
{
if (singleton_init() != 0) {
return 0;
}
return dict_v4_max_strokes(&dict_singleton);
}
void steno_dict_set_active(uint8_t dict_id)
{
if (singleton_init() != 0) {
return;
}
if (dict_id > 1 ||
!(dict_singleton.header->dicts_mask & (1u << dict_id))) {
return;
}
active_dict_id = dict_id;
}
#endif /* __ZEPHYR__ */

175
src/dict_v4.h Normal file
View file

@ -0,0 +1,175 @@
/*
* Copyright (c) 2024 zmk-steno-engine contributors
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*
* Licensed under the PolyForm Noncommercial License 1.0.0;
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* https://polyformproject.org/licenses/noncommercial/1.0.0
*/
/*
* Dictionary binary format v4 — "Union Split-Section" decoder.
*
* Byte layout per docs/FORMAT_V4.md. One union CHD MPHF over both
* source dicts (plover = 0, lapwing = 1), sections split across the
* two keyboard halves:
*
* LEFT: DISP, MEMBERSHIP, FP, CONFLICTS, VALIDX slice [0, k)
* RIGHT: STRDIR, STRINGS, CONFLICTS, VALIDX slice [k, n)
*
* The decision path (dict_v4_lookup) runs entirely on the left half;
* the string path (dict_v4_string_by_id_ctx / dict_v4_resolve_slot_ctx)
* runs on whichever context holds the needed sections.
*/
#ifndef DICT_V4_H
#define DICT_V4_H
#include <stdint.h>
#include <stddef.h>
#define DICT_V4_MAGIC 0x344E5453u /* "STN4" */
#define DICT_V4_VERSION 4
/* Section types (FORMAT_V4.md section 4) */
#define DICT_V4_SEC_DISP 1
#define DICT_V4_SEC_MEMBERSHIP 2
#define DICT_V4_SEC_VALIDX 3
#define DICT_V4_SEC_CONFLICTS 4
#define DICT_V4_SEC_FP 5
#define DICT_V4_SEC_STRDIR 6
#define DICT_V4_SEC_STRINGS 7
/* Source dictionary ids */
#define DICT_V4_DICT_PLOVER 0
#define DICT_V4_DICT_LAPWING 1
/* Displacement Huffman classes: class = bit_length(v), v <= 2^18-ish */
#define DICT_V4_DISP_CLASSES 19
/* VALIDX fixed field width */
#define DICT_V4_VALIDX_BITS 17
/* CONFLICTS record: u40le = slot (18 bits) | string_id << 18 */
#define DICT_V4_CONFLICT_REC_SIZE 5
/* Every translation is <= 255 bytes (compile-time enforced) */
#define DICT_V4_MAX_TEXT 255
/* FC string block decode buffer bound (FORMAT_V4.md section 4.7) */
#define DICT_V4_BLOCK_BUF_SIZE 16384
/* Local key buffer bound; header max_entry_strokes is the real limit */
#define DICT_V4_MAX_KEY_STROKES 16
/* dict_v4_lookup results (>= 0); negative values are -errno */
#define DICT_V4_MISS 0
#define DICT_V4_FOUND_LOCAL 1 /* string id known, in *string_id */
#define DICT_V4_FOUND_REMOTE 2 /* validx slice remote, caller RESOLVEs slot */
struct dict_v4_header {
uint32_t magic;
uint16_t version;
uint16_t section_count;
uint32_t n; /* union key count */
uint32_t bucket_count;
uint32_t string_count;
uint32_t conflict_count;
uint32_t d_threshold; /* displacement escape threshold */
uint8_t fp_bits; /* 4 */
uint8_t dicts_mask; /* bit0 plover, bit1 lapwing */
uint8_t max_entry_strokes;
uint8_t reserved;
} __attribute__((packed));
_Static_assert(sizeof(struct dict_v4_header) == 32, "header must be 32 bytes");
struct dict_v4_section {
uint8_t type;
uint8_t flags;
uint16_t rsvd;
uint32_t offset; /* relative to blob start */
uint32_t len;
uint32_t param; /* VALIDX: start_slot; STRDIR: block_count;
* CONFLICTS: record count */
} __attribute__((packed));
_Static_assert(sizeof(struct dict_v4_section) == 16, "dir entry must be 16 bytes");
struct dict_v4 {
const struct dict_v4_header *header;
/* DISP (NULL pointers = section absent from this half) */
const uint8_t *disp_code_len; /* u8[19] canonical code lengths */
const uint8_t *disp_skip; /* u32 LE entries, unaligned-safe reads */
uint32_t disp_skip_count;
const uint8_t *disp_stream; /* MSB-first Huffman bitstream */
uint32_t disp_stream_bits;
/* Canonical Huffman decode tables, built at init from code lengths.
* Indexed by code length (1..huff_max_len). */
uint8_t huff_max_len;
uint8_t huff_count[DICT_V4_DISP_CLASSES + 1];
uint32_t huff_first_code[DICT_V4_DISP_CLASSES + 1];
uint8_t huff_sym_base[DICT_V4_DISP_CLASSES + 1];
uint8_t huff_symbols[DICT_V4_DISP_CLASSES];
/* Fixed-width LSB-first packed sections */
const uint8_t *membership; /* 2 bits/slot */
const uint8_t *fp; /* fp_bits/slot */
const uint8_t *validx; /* 17 bits/slot slice */
uint32_t validx_start_slot;
uint32_t validx_slot_count;
/* CONFLICTS: sorted 5-byte records */
const uint8_t *conflicts;
uint32_t conflict_count;
/* String table */
const uint8_t *strdir_entries; /* {u32 comp_off; u32 first_string_id}[] */
uint32_t strdir_block_count;
const uint8_t *strings;
uint32_t strings_len;
};
/* Parse a half blob in place; no copies. Returns 0 or -errno. */
int dict_v4_init(struct dict_v4 *d, const void *blob, size_t len);
/*
* Decision path (left half). Returns DICT_V4_FOUND_LOCAL (string id
* known, in *string_id), DICT_V4_FOUND_REMOTE (validx slice remote,
* caller must RESOLVE *slot), DICT_V4_MISS, or -errno.
* *slot is set on both FOUND results.
*/
int dict_v4_lookup(const struct dict_v4 *d, const uint32_t *strokes, uint8_t count,
uint8_t active_dict, uint32_t *slot, uint32_t *string_id);
/*
* String path (right half or host test). Both return the translation
* length (>= 0, `out` NUL-terminated) or -errno.
*/
int dict_v4_string_by_id_ctx(const struct dict_v4 *d, uint32_t string_id,
char *out, size_t out_size);
int dict_v4_resolve_slot_ctx(const struct dict_v4 *d, uint32_t slot, uint8_t dict,
char *out, size_t out_size);
uint8_t dict_v4_max_strokes(const struct dict_v4 *d);
#ifdef __ZEPHYR__
/*
* Singleton wrappers over the blob embedded by dict_embed.S
* (steno_dict_start/steno_dict_end): left half gets steno_v4_left.bin,
* right half steno_v4_right.bin. Lazily initialized on first use.
*/
int dict_v4_string_by_id(uint32_t string_id, char *out, size_t out_size);
int dict_v4_resolve_slot(uint32_t slot, uint8_t dict, char *out, size_t out_size);
/* Dictionary dispatcher consumed by behavior_steno.c */
int steno_dict_lookup(const uint32_t *strokes, uint8_t count,
char *out, size_t out_size);
uint8_t steno_dict_max_strokes(void);
void steno_dict_set_active(uint8_t dict_id);
#endif /* __ZEPHYR__ */
#endif /* DICT_V4_H */

495
src/split_dict.c
View file

@ -8,214 +8,150 @@
* https://polyformproject.org/licenses/noncommercial/1.0.0
*/
#include <errno.h>
#include <string.h>
#include <zephyr/kernel.h>
#include <zephyr/bluetooth/bluetooth.h>
#include <zephyr/bluetooth/gatt.h>
#include <zephyr/bluetooth/conn.h>
#include <zephyr/bluetooth/uuid.h>
#include <zephyr/sys/byteorder.h>
#include <zephyr/logging/log.h>
#include "split_dict.h"
#include "split_cache.h"
#include "dict_mphf.h"
LOG_MODULE_REGISTER(split_dict, CONFIG_STENO_SPLIT_LOG_LEVEL);
/* Peripheral-side v4 dict resolvers, implemented in dict_v4.c */
extern int dict_v4_string_by_id(uint32_t string_id, char *out, size_t out_size);
extern int dict_v4_resolve_slot(uint32_t slot, uint8_t dict, char *out, size_t out_size);
/* Semaphore for blocking on BLE response */
static K_SEM_DEFINE(response_sem, 0, 1);
/* Current pending response state */
/* Current pending response state (central side) */
static uint8_t pending_seq;
static uint8_t response_buf[256];
static uint16_t response_len;
static uint8_t rx_buf[sizeof(struct steno_response_pkt) + SPLIT_DICT_MAX_TEXT];
static uint16_t rx_len;
static uint8_t seq_counter;
/* Notify buffer (peripheral side) */
static uint8_t notify_buf[sizeof(struct steno_response_pkt) + SPLIT_DICT_MAX_TEXT];
/* Cache instance */
static struct split_cache dict_cache;
/* Local MPHF dict for peripheral-side GATT lookups */
extern const uint8_t _steno_dict_start[];
extern const uint8_t _steno_dict_end[];
static struct dict_mphf peripheral_mphf;
static bool peripheral_dict_ready;
/* --- Cache key synthesis ---
*
* The LRU cache keys on stroke sequences (uint32_t[]). Protocol v4 requests
* are keyed instead by string_id or (slot, dict); synthesize a two-word
* pseudo-stroke key with a tag word that can never collide with a real
* 23-bit stroke bitmask. */
/* --- Helpers --- */
#define CACHE_TAG_GET_STRING 0x80000010u
#define CACHE_TAG_RESOLVE 0x80000011u
static void encode_strokes(const uint32_t *strokes, uint8_t count, uint8_t *out)
static void make_string_key(uint32_t string_id, uint32_t key[2])
{
for (uint8_t i = 0; i < count; i++) {
out[i * 3 + 0] = (strokes[i] >> 16) & 0xFF;
out[i * 3 + 1] = (strokes[i] >> 8) & 0xFF;
out[i * 3 + 2] = strokes[i] & 0xFF;
}
key[0] = CACHE_TAG_GET_STRING;
key[1] = string_id;
}
static void decode_strokes(const uint8_t *in, uint8_t count, uint32_t *strokes)
static void make_resolve_key(uint32_t slot, uint8_t dict, uint32_t key[2])
{
for (uint8_t i = 0; i < count; i++) {
strokes[i] = ((uint32_t)in[i * 3 + 0] << 16) |
((uint32_t)in[i * 3 + 1] << 8) |
(uint32_t)in[i * 3 + 2];
}
key[0] = CACHE_TAG_RESOLVE | ((uint32_t)dict << 8);
key[1] = slot;
}
/* --- GATT Write Callbacks (peripheral side handlers) --- */
static ssize_t dict_query_write_cb(struct bt_conn *conn,
const struct bt_gatt_attr *attr,
const void *buf, uint16_t len,
uint16_t offset, uint8_t flags)
static void send_response(struct bt_conn *conn, const struct bt_gatt_attr *attr,
uint8_t seq, int ret, const char *text)
{
const struct steno_query_pkt *pkt = buf;
struct steno_response_pkt *resp = (struct steno_response_pkt *)notify_buf;
uint16_t tlen = 0;
if (len < sizeof(struct steno_query_pkt)) {
LOG_WRN("Query pkt too short: %u", len);
return BT_GATT_ERR(BT_ATT_ERR_INVALID_ATTRIBUTE_LEN);
}
uint8_t stroke_count = pkt->stroke_count;
uint16_t expected = sizeof(struct steno_query_pkt) + stroke_count * 3;
if (len < expected) {
LOG_WRN("Query pkt truncated: got %u, need %u", len, expected);
return BT_GATT_ERR(BT_ATT_ERR_INVALID_ATTRIBUTE_LEN);
}
uint32_t strokes[8];
if (stroke_count > 8) {
stroke_count = 8;
}
decode_strokes(pkt->strokes, stroke_count, strokes);
/* Build response */
struct steno_response_pkt *resp = (struct steno_response_pkt *)response_buf;
resp->msg_type = STENO_MSG_RESPONSE;
resp->seq = pkt->seq;
resp->seq = seq;
const char *translation = NULL;
if (peripheral_dict_ready) {
translation = dict_mphf_lookup(&peripheral_mphf, strokes, stroke_count);
}
if (translation) {
uint16_t tlen = (uint16_t)strlen(translation);
resp->status = STENO_STATUS_FOUND;
resp->data_len = tlen;
memcpy(resp->data, translation, tlen);
response_len = sizeof(struct steno_response_pkt) + tlen;
} else {
if (ret >= 0) {
resp->status = STENO_STATUS_OK;
tlen = (uint16_t)strlen(text);
memcpy(resp->text, text, tlen);
} else if (ret == -ENOENT) {
resp->status = STENO_STATUS_NOT_FOUND;
resp->data_len = 0;
response_len = sizeof(struct steno_response_pkt);
} else {
resp->status = STENO_STATUS_ERROR;
}
/* Notify central with response */
bt_gatt_notify(conn, attr, response_buf, response_len);
resp->len = sys_cpu_to_le16(tlen);
bt_gatt_notify(conn, attr, notify_buf, sizeof(*resp) + tlen);
}
static ssize_t handle_get_string(struct bt_conn *conn,
const struct bt_gatt_attr *attr,
const void *buf, uint16_t len)
{
const struct steno_get_string_pkt *pkt = buf;
char text[SPLIT_DICT_MAX_TEXT + 1];
if (len < sizeof(*pkt)) {
LOG_WRN("GET_STRING pkt too short: %u", len);
return BT_GATT_ERR(BT_ATT_ERR_INVALID_ATTRIBUTE_LEN);
}
uint32_t string_id = sys_le32_to_cpu(pkt->string_id);
int ret = dict_v4_string_by_id(string_id, text, sizeof(text));
send_response(conn, attr, pkt->seq, ret, text);
return len;
}
static ssize_t dict_prefix_write_cb(struct bt_conn *conn,
static ssize_t handle_resolve(struct bt_conn *conn,
const struct bt_gatt_attr *attr,
const void *buf, uint16_t len,
uint16_t offset, uint8_t flags)
const void *buf, uint16_t len)
{
const struct steno_query_pkt *pkt = buf;
const struct steno_resolve_pkt *pkt = buf;
char text[SPLIT_DICT_MAX_TEXT + 1];
if (len < sizeof(struct steno_query_pkt)) {
if (len < sizeof(*pkt)) {
LOG_WRN("RESOLVE pkt too short: %u", len);
return BT_GATT_ERR(BT_ATT_ERR_INVALID_ATTRIBUTE_LEN);
}
uint8_t stroke_count = pkt->stroke_count;
if (stroke_count > 8) {
stroke_count = 8;
}
uint32_t slot = sys_le32_to_cpu(pkt->slot);
uint32_t strokes[8];
decode_strokes(pkt->strokes, stroke_count, strokes);
int ret = dict_v4_resolve_slot(slot, pkt->dict, text, sizeof(text));
struct steno_response_pkt *resp = (struct steno_response_pkt *)response_buf;
resp->msg_type = STENO_MSG_RESPONSE;
resp->seq = pkt->seq;
resp->data_len = 0;
if (peripheral_dict_ready &&
stroke_count == 1 &&
dict_mphf_has_prefix(&peripheral_mphf, strokes[0])) {
resp->status = STENO_STATUS_PREFIX_ONLY;
} else {
resp->status = STENO_STATUS_NOT_FOUND;
}
response_len = sizeof(struct steno_response_pkt);
bt_gatt_notify(conn, attr, response_buf, response_len);
send_response(conn, attr, pkt->seq, ret, text);
return len;
}
static ssize_t dict_batch_write_cb(struct bt_conn *conn,
static ssize_t dict_req_write_cb(struct bt_conn *conn,
const struct bt_gatt_attr *attr,
const void *buf, uint16_t len,
uint16_t offset, uint8_t flags)
{
const struct steno_batch_query_pkt *pkt = buf;
const uint8_t *bytes = buf;
if (len < sizeof(struct steno_batch_query_pkt)) {
if (len < 1) {
return BT_GATT_ERR(BT_ATT_ERR_INVALID_ATTRIBUTE_LEN);
}
LOG_DBG("Batch query: %u queries", pkt->query_count);
/* Process each sub-query packed in queries[] */
uint16_t pos = 0;
const uint8_t *data = pkt->queries;
uint16_t data_len = len - sizeof(struct steno_batch_query_pkt);
for (uint8_t q = 0; q < pkt->query_count && pos < data_len; q++) {
if (pos >= data_len) {
break;
switch (bytes[0]) {
case STENO_MSG_GET_STRING:
return handle_get_string(conn, attr, buf, len);
case STENO_MSG_RESOLVE:
return handle_resolve(conn, attr, buf, len);
default:
LOG_WRN("Unknown msg type: 0x%02x", bytes[0]);
return BT_GATT_ERR(BT_ATT_ERR_NOT_SUPPORTED);
}
uint8_t stroke_count = data[pos];
pos++;
if (stroke_count > 8) {
stroke_count = 8;
}
if (pos + stroke_count * 3 > data_len) {
break;
}
uint32_t strokes[8];
decode_strokes(&data[pos], stroke_count, strokes);
pos += stroke_count * 3;
/* Lookup and send individual response per query */
struct steno_response_pkt *resp = (struct steno_response_pkt *)response_buf;
resp->msg_type = STENO_MSG_RESPONSE;
resp->seq = pkt->seq;
const char *translation = NULL;
if (peripheral_dict_ready) {
translation = dict_mphf_lookup(&peripheral_mphf, strokes, stroke_count);
}
if (translation) {
uint16_t tlen = (uint16_t)strlen(translation);
resp->status = STENO_STATUS_FOUND;
resp->data_len = tlen;
memcpy(resp->data, translation, tlen);
response_len = sizeof(struct steno_response_pkt) + tlen;
} else {
resp->status = STENO_STATUS_NOT_FOUND;
resp->data_len = 0;
response_len = sizeof(struct steno_response_pkt);
}
bt_gatt_notify(conn, attr, response_buf, response_len);
}
return len;
}
/* --- Notification callback (central side) --- */
@ -236,9 +172,12 @@ static uint8_t notify_cb(struct bt_conn *conn,
return BT_GATT_ITER_CONTINUE;
}
if (resp->seq == pending_seq) {
memcpy(response_buf, data, length);
response_len = length;
if (resp->msg_type == STENO_MSG_RESPONSE && resp->seq == pending_seq) {
if (length > sizeof(rx_buf)) {
length = sizeof(rx_buf);
}
memcpy(rx_buf, data, length);
rx_len = length;
k_sem_give(&response_sem);
}
@ -250,25 +189,11 @@ static uint8_t notify_cb(struct bt_conn *conn,
BT_GATT_SERVICE_DEFINE(steno_dict_svc,
BT_GATT_PRIMARY_SERVICE(STENO_UUID_SERVICE),
/* Dict Query characteristic: write + notify */
BT_GATT_CHARACTERISTIC(STENO_UUID_DICT_QUERY,
/* Dict request characteristic: write + notify */
BT_GATT_CHARACTERISTIC(STENO_UUID_DICT_REQ,
BT_GATT_CHRC_WRITE | BT_GATT_CHRC_NOTIFY,
BT_GATT_PERM_WRITE,
NULL, dict_query_write_cb, NULL),
BT_GATT_CCC(NULL, BT_GATT_PERM_READ | BT_GATT_PERM_WRITE),
/* Dict Prefix characteristic: write + notify */
BT_GATT_CHARACTERISTIC(STENO_UUID_DICT_PREFIX,
BT_GATT_CHRC_WRITE | BT_GATT_CHRC_NOTIFY,
BT_GATT_PERM_WRITE,
NULL, dict_prefix_write_cb, NULL),
BT_GATT_CCC(NULL, BT_GATT_PERM_READ | BT_GATT_PERM_WRITE),
/* Dict Batch characteristic: write + notify */
BT_GATT_CHARACTERISTIC(STENO_UUID_DICT_BATCH,
BT_GATT_CHRC_WRITE | BT_GATT_CHRC_NOTIFY,
BT_GATT_PERM_WRITE,
NULL, dict_batch_write_cb, NULL),
NULL, dict_req_write_cb, NULL),
BT_GATT_CCC(NULL, BT_GATT_PERM_READ | BT_GATT_PERM_WRITE),
);
@ -278,183 +203,137 @@ BT_GATT_SERVICE_DEFINE(steno_dict_svc,
static struct bt_conn *split_conn;
static struct bt_gatt_subscribe_params subscribe_params;
int split_dict_lookup(const uint32_t *strokes, uint8_t count,
char *result, size_t result_size)
/* Send one request packet, block for its RESPONSE, copy text into out.
* Returns text length on success, -ENOENT on NOT_FOUND, negative errno
* otherwise. out must hold SPLIT_DICT_MAX_TEXT + 1 bytes. */
static int split_request(const void *pkt, uint16_t pkt_len, uint8_t seq, char *out)
{
if (count == 0 || count > 8) {
return -EINVAL;
}
/* Check cache first */
bool has_prefix;
if (split_cache_lookup(&dict_cache, strokes, count, result, result_size, &has_prefix)) {
LOG_DBG("Cache hit for %u strokes", count);
return strlen(result);
}
int err;
if (!split_conn) {
LOG_ERR("No split connection");
return -ENOTCONN;
}
/* Build query packet */
uint8_t pkt_buf[sizeof(struct steno_query_pkt) + 8 * 3];
struct steno_query_pkt *pkt = (struct steno_query_pkt *)pkt_buf;
pkt->msg_type = STENO_MSG_QUERY;
pkt->seq = seq_counter++;
pkt->stroke_count = count;
encode_strokes(strokes, count, pkt->strokes);
pending_seq = pkt->seq;
pending_seq = seq;
k_sem_reset(&response_sem);
uint16_t pkt_len = sizeof(struct steno_query_pkt) + count * 3;
/* Send via GATT write */
int err = bt_gatt_write_without_response(split_conn, 0, pkt_buf, pkt_len, false);
err = bt_gatt_write_without_response(split_conn, 0, pkt, pkt_len, false);
if (err) {
LOG_ERR("GATT write failed: %d", err);
return err;
}
/* Wait for response */
err = k_sem_take(&response_sem, K_MSEC(CONFIG_STENO_SPLIT_TIMEOUT_MS));
if (err) {
LOG_WRN("Response timeout");
return -ETIMEDOUT;
}
/* Decode response */
const struct steno_response_pkt *resp = (const struct steno_response_pkt *)response_buf;
if (resp->status == STENO_STATUS_FOUND) {
uint16_t copy_len = resp->data_len;
if (copy_len >= result_size) {
copy_len = result_size - 1;
}
memcpy(result, resp->data, copy_len);
result[copy_len] = '\0';
/* Cache the result */
split_cache_insert(&dict_cache, strokes, count, result, false);
return copy_len;
}
const struct steno_response_pkt *resp = (const struct steno_response_pkt *)rx_buf;
if (resp->status == STENO_STATUS_NOT_FOUND) {
return -ENOENT;
}
if (resp->status != STENO_STATUS_OK) {
return -EIO;
}
bool split_dict_has_prefix(const uint32_t *strokes, uint8_t count)
uint16_t tlen = sys_le16_to_cpu(resp->len);
uint16_t avail = rx_len - sizeof(struct steno_response_pkt);
if (tlen > avail) {
LOG_WRN("Response truncated: len %u, got %u", tlen, avail);
return -EIO;
}
memcpy(out, resp->text, tlen);
out[tlen] = '\0';
return tlen;
}
/* Copy full text into caller buffer, truncating if needed */
static int copy_out(const char *text, uint16_t tlen, char *out, size_t out_size)
{
if (count == 0 || count > 8) {
return false;
size_t copy_len = tlen;
if (copy_len >= out_size) {
copy_len = out_size - 1;
}
memcpy(out, text, copy_len);
out[copy_len] = '\0';
return (int)copy_len;
}
/* Check cache */
bool has_prefix;
char dummy[1];
if (split_cache_lookup(&dict_cache, strokes, count, dummy, sizeof(dummy), &has_prefix)) {
return has_prefix;
}
if (!split_conn) {
return false;
}
uint8_t pkt_buf[sizeof(struct steno_query_pkt) + 8 * 3];
struct steno_query_pkt *pkt = (struct steno_query_pkt *)pkt_buf;
pkt->msg_type = STENO_MSG_PREFIX;
pkt->seq = seq_counter++;
pkt->stroke_count = count;
encode_strokes(strokes, count, pkt->strokes);
pending_seq = pkt->seq;
k_sem_reset(&response_sem);
uint16_t pkt_len = sizeof(struct steno_query_pkt) + count * 3;
int err = bt_gatt_write_without_response(split_conn, 0, pkt_buf, pkt_len, false);
if (err) {
return false;
}
err = k_sem_take(&response_sem, K_MSEC(CONFIG_STENO_SPLIT_TIMEOUT_MS));
if (err) {
return false;
}
const struct steno_response_pkt *resp = (const struct steno_response_pkt *)response_buf;
return resp->status == STENO_STATUS_PREFIX_ONLY;
}
int split_dict_batch_lookup(const uint32_t **stroke_seqs, const uint8_t *counts,
uint8_t num_queries, struct steno_batch_result *results)
int split_dict_get_string(uint32_t string_id, char *out, size_t out_size)
{
if (num_queries == 0 || !split_conn) {
if (!out || out_size == 0) {
return -EINVAL;
}
/* Build batch packet */
uint8_t pkt_buf[256];
struct steno_batch_query_pkt *pkt = (struct steno_batch_query_pkt *)pkt_buf;
uint32_t key[2];
make_string_key(string_id, key);
pkt->msg_type = STENO_MSG_BATCH;
pkt->seq = seq_counter++;
pkt->query_count = num_queries;
uint16_t pos = 0;
for (uint8_t q = 0; q < num_queries; q++) {
uint8_t cnt = counts[q];
if (cnt > 8) {
cnt = 8;
}
pkt->queries[pos] = cnt;
pos++;
encode_strokes(stroke_seqs[q], cnt, &pkt->queries[pos]);
pos += cnt * 3;
if (split_cache_lookup(&dict_cache, key, 2, out, out_size, NULL)) {
LOG_DBG("Cache hit for string %u", string_id);
return strlen(out);
}
uint16_t pkt_len = sizeof(struct steno_batch_query_pkt) + pos;
struct steno_get_string_pkt pkt = {
.msg_type = STENO_MSG_GET_STRING,
.seq = seq_counter++,
.string_id = sys_cpu_to_le32(string_id),
};
pending_seq = pkt->seq;
k_sem_reset(&response_sem);
int err = bt_gatt_write_without_response(split_conn, 0, pkt_buf, pkt_len, false);
if (err) {
return err;
char text[SPLIT_DICT_MAX_TEXT + 1];
int ret = split_request(&pkt, sizeof(pkt), pkt.seq, text);
if (ret < 0) {
return ret;
}
/* Collect responses for each query */
for (uint8_t q = 0; q < num_queries; q++) {
err = k_sem_take(&response_sem, K_MSEC(CONFIG_STENO_SPLIT_TIMEOUT_MS));
if (err) {
results[q].status = STENO_STATUS_ERROR;
continue;
/* Cache only entries that fit the cache value slot untruncated */
if ((size_t)ret < SPLIT_CACHE_VALUE_SIZE) {
split_cache_insert(&dict_cache, key, 2, text, false);
}
const struct steno_response_pkt *resp =
(const struct steno_response_pkt *)response_buf;
results[q].status = resp->status;
results[q].has_prefix = (resp->status == STENO_STATUS_PREFIX_ONLY);
if (resp->status == STENO_STATUS_FOUND && resp->data_len > 0) {
uint16_t copy_len = resp->data_len;
if (copy_len >= sizeof(results[q].translation)) {
copy_len = sizeof(results[q].translation) - 1;
}
memcpy(results[q].translation, resp->data, copy_len);
results[q].translation[copy_len] = '\0';
results[q].translation_len = copy_len;
} else {
results[q].translation[0] = '\0';
results[q].translation_len = 0;
}
return copy_out(text, ret, out, out_size);
}
return 0;
int split_dict_resolve(uint32_t slot, uint8_t dict, char *out, size_t out_size)
{
if (!out || out_size == 0) {
return -EINVAL;
}
uint32_t key[2];
make_resolve_key(slot, dict, key);
if (split_cache_lookup(&dict_cache, key, 2, out, out_size, NULL)) {
LOG_DBG("Cache hit for slot %u dict %u", slot, dict);
return strlen(out);
}
struct steno_resolve_pkt pkt = {
.msg_type = STENO_MSG_RESOLVE,
.seq = seq_counter++,
.slot = sys_cpu_to_le32(slot),
.dict = dict,
};
char text[SPLIT_DICT_MAX_TEXT + 1];
int ret = split_request(&pkt, sizeof(pkt), pkt.seq, text);
if (ret < 0) {
return ret;
}
/* Cache only entries that fit the cache value slot untruncated */
if ((size_t)ret < SPLIT_CACHE_VALUE_SIZE) {
split_cache_insert(&dict_cache, key, 2, text, false);
}
return copy_out(text, ret, out, out_size);
}
int split_dict_init(void)
@ -462,20 +341,10 @@ int split_dict_init(void)
split_cache_init(&dict_cache);
seq_counter = 0;
split_conn = NULL;
subscribe_params.notify = notify_cb;
subscribe_params.value = BT_GATT_CCC_NOTIFY;
/* Init peripheral-side MPHF dict for GATT lookups */
size_t dict_size = _steno_dict_end - _steno_dict_start;
if (dict_size > 4) {
int ret = dict_mphf_init(&peripheral_mphf, _steno_dict_start, dict_size);
if (ret == 0) {
peripheral_dict_ready = true;
LOG_INF("Peripheral partition loaded (%u bytes)", (unsigned)dict_size);
} else {
LOG_ERR("Peripheral partition init failed: %d", ret);
}
}
LOG_INF("Split dict initialized");
LOG_INF("Split dict initialized (protocol v4)");
return 0;
}

80
src/split_dict.h
View file

@ -11,7 +11,6 @@
#ifndef SPLIT_DICT_H
#define SPLIT_DICT_H
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <zephyr/bluetooth/uuid.h>
@ -26,72 +25,53 @@
BT_UUID_DECLARE_128(BT_UUID_128_ENCODE( \
0x7374656e, 0x6f00, 0x4000, 0x8000, 0x000000000001))
#define STENO_UUID_DICT_QUERY \
#define STENO_UUID_DICT_REQ \
BT_UUID_DECLARE_128(BT_UUID_128_ENCODE( \
0x7374656e, 0x6f00, 0x4000, 0x8000, 0x000000000002))
#define STENO_UUID_DICT_PREFIX \
BT_UUID_DECLARE_128(BT_UUID_128_ENCODE( \
0x7374656e, 0x6f00, 0x4000, 0x8000, 0x000000000003))
#define STENO_UUID_DICT_BATCH \
BT_UUID_DECLARE_128(BT_UUID_128_ENCODE( \
0x7374656e, 0x6f00, 0x4000, 0x8000, 0x000000000004))
/* Message types */
/* BLE protocol v4 message types (FORMAT_V4.md section 7) */
enum steno_msg_type {
STENO_MSG_QUERY = 0x01,
STENO_MSG_PREFIX = 0x02,
STENO_MSG_BATCH = 0x03,
STENO_MSG_GET_STRING = 0x10,
STENO_MSG_RESOLVE = 0x11,
STENO_MSG_RESPONSE = 0x80,
};
/* Status codes */
/* Response status codes */
enum steno_status {
STENO_STATUS_FOUND = 0,
STENO_STATUS_OK = 0,
STENO_STATUS_NOT_FOUND = 1,
STENO_STATUS_PREFIX_ONLY = 2,
STENO_STATUS_ERROR = 3,
STENO_STATUS_ERROR = 2,
};
/* Packet structures */
struct steno_query_pkt {
uint8_t msg_type;
/* Format v4 guarantees every translation <= 255 bytes */
#define SPLIT_DICT_MAX_TEXT 255
/* Packet structures; all multi-byte fields little-endian on the wire */
struct steno_get_string_pkt {
uint8_t msg_type; /* STENO_MSG_GET_STRING */
uint8_t seq;
uint8_t stroke_count;
uint8_t strokes[]; /* 3 bytes per stroke (24-bit packed) */
} __packed;
uint32_t string_id; /* LE */
} __attribute__((packed));
struct steno_resolve_pkt {
uint8_t msg_type; /* STENO_MSG_RESOLVE */
uint8_t seq;
uint32_t slot; /* LE */
uint8_t dict; /* 0 = plover, 1 = lapwing */
} __attribute__((packed));
struct steno_response_pkt {
uint8_t msg_type;
uint8_t msg_type; /* STENO_MSG_RESPONSE */
uint8_t seq;
uint8_t status;
uint16_t data_len;
uint8_t data[]; /* translation string (UTF-8, not null-terminated) */
} __packed;
uint8_t status; /* enum steno_status */
uint16_t len; /* LE, byte length of text[] */
char text[]; /* translation (UTF-8, not null-terminated) */
} __attribute__((packed));
struct steno_batch_query_pkt {
uint8_t msg_type;
uint8_t seq;
uint8_t query_count;
uint8_t queries[]; /* packed steno_query_pkt entries (without msg_type/seq) */
} __packed;
/* Batch result entry */
struct steno_batch_result {
uint8_t status;
char translation[128];
uint16_t translation_len;
bool has_prefix;
};
/* API */
/* Central-side API */
int split_dict_init(void);
int split_dict_lookup(const uint32_t *strokes, uint8_t count,
char *result, size_t result_size);
bool split_dict_has_prefix(const uint32_t *strokes, uint8_t count);
int split_dict_batch_lookup(const uint32_t **stroke_seqs, const uint8_t *counts,
uint8_t num_queries, struct steno_batch_result *results);
int split_dict_get_string(uint32_t string_id, char *out, size_t out_size);
int split_dict_resolve(uint32_t slot, uint8_t dict, char *out, size_t out_size);
/* GATT service registration */
int split_dict_gatt_register(void);

11
src/undo.c
View file

@ -58,6 +58,17 @@ const struct stroke_history_entry *steno_undo_peek(const struct stroke_history *
return &hist->entries[idx];
}
const struct stroke_history_entry *steno_undo_peek_at(const struct stroke_history *hist,
uint16_t back)
{
if (back >= hist->count) {
return NULL;
}
uint16_t idx = (hist->head + CONFIG_STENO_HISTORY_SIZE - 1 - back) % CONFIG_STENO_HISTORY_SIZE;
return &hist->entries[idx];
}
uint16_t steno_undo_count(const struct stroke_history *hist)
{
return hist->count;

5
src/undo.h
View file

@ -38,6 +38,11 @@ struct stroke_history_entry *steno_undo_pop(struct stroke_history *hist);
/* Peek at most recent entry without removing. Returns NULL if empty. */
const struct stroke_history_entry *steno_undo_peek(const struct stroke_history *hist);
/* Peek `back` entries behind the most recent (0 = most recent).
* Returns NULL if out of range. */
const struct stroke_history_entry *steno_undo_peek_at(const struct stroke_history *hist,
uint16_t back);
/* Get current count */
uint16_t steno_undo_count(const struct stroke_history *hist);

267
tests/test_dict_v4.c Normal file
View file

@ -0,0 +1,267 @@
/*
* Copyright (c) 2024 zmk-steno-engine contributors
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*
* Licensed under the PolyForm Noncommercial License 1.0.0;
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* https://polyformproject.org/licenses/noncommercial/1.0.0
*/
/*
* Host round-trip test for the format v4 decoder.
*
* Loads the two half blobs produced by tools/compile_v4.py plus a
* vector file of (dict_id, strokes, expected translation) records,
* then drives the real split decision path: every lookup runs on the
* LEFT context; the string is fetched from the RIGHT context via
* string_by_id (FOUND_LOCAL) or resolve_slot (FOUND_REMOTE) and
* compared byte-exact. Unknown keys must MISS, modulo the 4-bit
* fingerprint false-accept rate (asserted < 10%).
*
* test_dict_v4 <left.bin> <right.bin> <vectors.bin>
*
* Vector file layout: u32 magic 0x56543456, u32 count, then records
* { u8 dict_id; u8 stroke_count; u8 known; u8 rsvd;
* u32 strokes[stroke_count]; u16 t_len; u8 translation[t_len]; }.
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stdint.h>
#include "dict_v4.h"
#define VECTORS_MAGIC 0x56543456u
#define MAX_MISMATCH_PRINT 10
#define FALSE_ACCEPT_LIMIT_PCT 10.0
static uint8_t *load_file(const char *path, size_t *len_out)
{
FILE *f = fopen(path, "rb");
if (!f) {
fprintf(stderr, "FAIL: cannot open %s\n", path);
return NULL;
}
fseek(f, 0, SEEK_END);
long len = ftell(f);
fseek(f, 0, SEEK_SET);
if (len < 0) {
fclose(f);
return NULL;
}
uint8_t *buf = malloc((size_t)len);
if (!buf || fread(buf, 1, (size_t)len, f) != (size_t)len) {
fprintf(stderr, "FAIL: cannot read %s\n", path);
free(buf);
fclose(f);
return NULL;
}
fclose(f);
*len_out = (size_t)len;
return buf;
}
static uint32_t get_le32(const uint8_t *p)
{
return (uint32_t)p[0] | ((uint32_t)p[1] << 8)
| ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24);
}
static uint16_t get_le16(const uint8_t *p)
{
return (uint16_t)(p[0] | (p[1] << 8));
}
static void print_strokes(const uint32_t *strokes, uint8_t count)
{
for (uint8_t i = 0; i < count; i++) {
fprintf(stderr, "%s0x%06x", i ? "/" : "", strokes[i]);
}
}
int main(int argc, char **argv)
{
if (argc != 4) {
fprintf(stderr, "usage: %s <left.bin> <right.bin> <vectors.bin>\n",
argv[0]);
return 2;
}
size_t left_len, right_len, vec_len;
uint8_t *left_blob = load_file(argv[1], &left_len);
uint8_t *right_blob = load_file(argv[2], &right_len);
uint8_t *vec = load_file(argv[3], &vec_len);
if (!left_blob || !right_blob || !vec) {
return 2;
}
struct dict_v4 left, right;
int ret = dict_v4_init(&left, left_blob, left_len);
if (ret != 0) {
fprintf(stderr, "FAIL: dict_v4_init(left) = %d\n", ret);
return 1;
}
ret = dict_v4_init(&right, right_blob, right_len);
if (ret != 0) {
fprintf(stderr, "FAIL: dict_v4_init(right) = %d\n", ret);
return 1;
}
if (left.header->n != right.header->n ||
left.header->string_count != right.header->string_count ||
left.header->conflict_count != right.header->conflict_count) {
fprintf(stderr, "FAIL: left/right header mismatch\n");
return 1;
}
if (left.validx_start_slot != 0 ||
left.validx_slot_count != right.validx_start_slot ||
right.validx_start_slot + right.validx_slot_count != left.header->n) {
fprintf(stderr, "FAIL: VALIDX slices do not cover [0, n) "
"(left [%u, %u), right [%u, %u))\n",
left.validx_start_slot,
left.validx_start_slot + left.validx_slot_count,
right.validx_start_slot,
right.validx_start_slot + right.validx_slot_count);
return 1;
}
if (dict_v4_max_strokes(&left) == 0) {
fprintf(stderr, "FAIL: max_strokes = 0\n");
return 1;
}
if (vec_len < 8 || get_le32(vec) != VECTORS_MAGIC) {
fprintf(stderr, "FAIL: bad vector file magic\n");
return 2;
}
uint32_t vec_count = get_le32(vec + 4);
uint32_t known_checked = 0, unknown_checked = 0;
uint32_t mismatches = 0, false_accepts = 0;
uint32_t local_hits = 0, remote_hits = 0;
size_t pos = 8;
for (uint32_t rec = 0; rec < vec_count; rec++) {
if (pos + 4 > vec_len) {
fprintf(stderr, "FAIL: truncated vector file (record %u)\n", rec);
return 2;
}
uint8_t dict_id = vec[pos];
uint8_t stroke_count = vec[pos + 1];
uint8_t known = vec[pos + 2];
pos += 4;
if (stroke_count == 0 || stroke_count > DICT_V4_MAX_KEY_STROKES ||
pos + (size_t)stroke_count * 4 + 2 > vec_len) {
fprintf(stderr, "FAIL: bad vector record %u\n", rec);
return 2;
}
uint32_t strokes[DICT_V4_MAX_KEY_STROKES];
for (uint8_t i = 0; i < stroke_count; i++) {
strokes[i] = get_le32(vec + pos + (size_t)i * 4);
}
pos += (size_t)stroke_count * 4;
uint16_t t_len = get_le16(vec + pos);
pos += 2;
if (pos + t_len > vec_len) {
fprintf(stderr, "FAIL: truncated translation (record %u)\n", rec);
return 2;
}
const uint8_t *expected = vec + pos;
pos += t_len;
/* Decision path always on the LEFT context */
uint32_t slot = 0, string_id = 0;
ret = dict_v4_lookup(&left, strokes, stroke_count, dict_id,
&slot, &string_id);
char text[DICT_V4_MAX_TEXT + 1];
int text_len = -1;
/* String path on the RIGHT context */
if (ret == DICT_V4_FOUND_LOCAL) {
local_hits++;
text_len = dict_v4_string_by_id_ctx(&right, string_id,
text, sizeof(text));
} else if (ret == DICT_V4_FOUND_REMOTE) {
remote_hits++;
text_len = dict_v4_resolve_slot_ctx(&right, slot, dict_id,
text, sizeof(text));
}
if (known) {
known_checked++;
int ok = (ret == DICT_V4_FOUND_LOCAL ||
ret == DICT_V4_FOUND_REMOTE) &&
text_len == (int)t_len &&
memcmp(text, expected, t_len) == 0;
if (!ok) {
mismatches++;
if (mismatches <= MAX_MISMATCH_PRINT) {
fprintf(stderr, "MISMATCH dict=%u strokes=", dict_id);
print_strokes(strokes, stroke_count);
fprintf(stderr, " lookup=%d text_len=%d expected=%.*s",
ret, text_len, (int)t_len, (const char *)expected);
if (text_len >= 0) {
fprintf(stderr, " got=%.*s", text_len, text);
}
fprintf(stderr, "\n");
}
}
} else {
unknown_checked++;
if (ret == DICT_V4_FOUND_LOCAL || ret == DICT_V4_FOUND_REMOTE) {
/* Fingerprint false accept: string fetch must still
* succeed (no crash, valid id), the text is garbage. */
false_accepts++;
if (text_len < 0) {
mismatches++;
if (mismatches <= MAX_MISMATCH_PRINT) {
fprintf(stderr, "BAD FALSE-ACCEPT dict=%u lookup=%d "
"string fetch=%d strokes=",
dict_id, ret, text_len);
print_strokes(strokes, stroke_count);
fprintf(stderr, "\n");
}
}
} else if (ret != DICT_V4_MISS) {
mismatches++;
if (mismatches <= MAX_MISMATCH_PRINT) {
fprintf(stderr, "BAD MISS ret=%d strokes=", ret);
print_strokes(strokes, stroke_count);
fprintf(stderr, "\n");
}
}
}
}
double fa_pct = unknown_checked
? 100.0 * false_accepts / unknown_checked : 0.0;
printf("vectors: %u (known %u, unknown %u)\n",
vec_count, known_checked, unknown_checked);
printf("hits: local %u, remote %u; mismatches %u\n",
local_hits, remote_hits, mismatches);
printf("false accepts: %u / %u (%.2f%%)\n",
false_accepts, unknown_checked, fa_pct);
if (mismatches != 0) {
printf("FAIL: %u mismatches\n", mismatches);
return 1;
}
if (fa_pct >= FALSE_ACCEPT_LIMIT_PCT) {
printf("FAIL: false-accept rate %.2f%% >= %.1f%%\n",
fa_pct, FALSE_ACCEPT_LIMIT_PCT);
return 1;
}
printf("PASS\n");
free(left_blob);
free(right_blob);
free(vec);
return 0;
}

1353
tools/compile_v4.py Normal file
View file

File diff suppressed because it is too large Load diff