v4 union split-section dictionary: both full dicts, zero trimming
Format v4 (docs/FORMAT_V4.md): one union CHD MPHF over 226,791 stroke keys from Plover (147,424) + Lapwing (114,885), sections distributed across halves. Left blob 387KB (displacements log-class Huffman 59KB, membership 57KB, fingerprints 113KB, conflicts 9.6KB, value-index slice 157KB); right blob 515KB (strings front-coded+deflate 193KB, value-index remainder 324KB, conflicts dup). Shared string table: 73,464 unique translations, Lapwing 93% subset of Plover. - tools/compile_v4.py: encoder + full verification (all 262,309 entries byte-exact through cold decode path; hard error if either blob exceeds budget — never trims) - src/dict_v4.c/.h: decoder — canonical Huffman displacement decode, conflict binary search, FC block walk, 2-block LRU cache - src/behavior_steno.c: Plover-style sliding longest-match with retrace replaces multi-stroke timeout + prefix checks - src/split_dict.c/.h: protocol v4 — GET_STRING(string_id) / RESOLVE(slot, dict); decisions are 100% left-local, BLE only for translation text - CMake/Kconfig: STENO_DICT_BOTH default, one compiler run emits both half blobs; legacy MPHF path kept behind STENO_DICT_V4=n - Host round-trip test: 42,000 vectors left→right split path, 0 mismatches (tests/test_dict_v4.c)
This commit is contained in:
parent
dd593c1640
commit
9bf21c9db2
12 changed files with 3338 additions and 666 deletions
271
CMakeLists.txt
271
CMakeLists.txt
|
|
@ -8,108 +8,179 @@ target_include_directories(app PRIVATE
|
|||
${CMAKE_CURRENT_SOURCE_DIR}/src
|
||||
)
|
||||
|
||||
# ── Split-dict mode ──────────────────────────────────
|
||||
# TRUE split: both halves have local MPHF partition + BLE fallback
|
||||
# Central: behavior engine + local left partition + BLE client
|
||||
# Peripheral: local right partition + GATT server
|
||||
if(CONFIG_STENO_SPLIT_DICT)
|
||||
find_package(Python3 REQUIRED COMPONENTS Interpreter)
|
||||
|
||||
if(CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
|
||||
# Central: behavior engine + local dict + BLE client for remote partition
|
||||
target_sources(app PRIVATE
|
||||
src/behavior_steno.c
|
||||
src/output.c
|
||||
src/formatter.c
|
||||
src/undo.c
|
||||
src/dict_embed.S
|
||||
src/dict_mphf.c
|
||||
src/split_dict.c
|
||||
src/split_cache.c
|
||||
)
|
||||
else()
|
||||
# Peripheral: local dict + GATT server for central queries
|
||||
target_sources(app PRIVATE
|
||||
src/dict_embed.S
|
||||
src/dict_mphf.c
|
||||
src/split_dict.c
|
||||
src/split_cache.c
|
||||
)
|
||||
endif()
|
||||
set(STENO_DICTS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/dicts)
|
||||
set(STENO_FETCH ${CMAKE_CURRENT_SOURCE_DIR}/tools/fetch_dict.py)
|
||||
|
||||
# ── Non-split mode ───────────────────────────────────
|
||||
# Everything on one board (or central-only without split)
|
||||
# ── Dictionary selection ─────────────────────────────
|
||||
# Maps the Kconfig choice onto the compiler's --dicts argument.
|
||||
if(CONFIG_STENO_DICT_PLOVER)
|
||||
set(STENO_DICTS_ARG "plover")
|
||||
elseif(CONFIG_STENO_DICT_LAPWING)
|
||||
set(STENO_DICTS_ARG "lapwing")
|
||||
else()
|
||||
if(NOT CONFIG_ZMK_SPLIT OR CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
|
||||
target_sources(app PRIVATE
|
||||
src/behavior_steno.c
|
||||
src/output.c
|
||||
src/formatter.c
|
||||
src/undo.c
|
||||
src/dict_embed.S
|
||||
set(STENO_DICTS_ARG "both")
|
||||
endif()
|
||||
|
||||
# Ensure a dictionary JSON is present: download at configure time if
|
||||
# missing (hard error on failure), and add a build-time stamp rule that
|
||||
# re-checks the upstream hash on fresh build directories.
|
||||
function(steno_fetch_dict name json_path)
|
||||
if(NOT EXISTS ${json_path})
|
||||
message(STATUS "Steno: downloading ${name} dictionary...")
|
||||
execute_process(
|
||||
COMMAND ${Python3_EXECUTABLE} ${STENO_FETCH} ${name} ${STENO_DICTS_DIR}
|
||||
RESULT_VARIABLE FETCH_RESULT
|
||||
)
|
||||
if(CONFIG_STENO_DICT_MPHF)
|
||||
target_sources(app PRIVATE src/dict_mphf.c)
|
||||
else()
|
||||
target_sources(app PRIVATE src/trie.c)
|
||||
if(NOT FETCH_RESULT EQUAL 0 OR NOT EXISTS ${json_path})
|
||||
message(FATAL_ERROR
|
||||
"Steno: ${name} dictionary is missing and the download failed "
|
||||
"(expected ${json_path}).")
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
add_custom_command(
|
||||
OUTPUT ${json_path}.stamp
|
||||
COMMAND ${Python3_EXECUTABLE} ${STENO_FETCH} ${name} ${STENO_DICTS_DIR}
|
||||
COMMAND ${CMAKE_COMMAND} -E touch ${json_path}.stamp
|
||||
COMMENT "Checking ${name} dictionary for updates"
|
||||
)
|
||||
endfunction()
|
||||
|
||||
# ── Dictionary compilation ───────────────────────────
|
||||
# Both halves need dict embed in split mode; non-split same as before
|
||||
set(STENO_NEED_DICT_EMBED FALSE)
|
||||
if(CONFIG_STENO_SPLIT_DICT)
|
||||
# Both central and peripheral need their own partition
|
||||
set(STENO_NEED_DICT_EMBED TRUE)
|
||||
elseif(NOT CONFIG_ZMK_SPLIT OR CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
|
||||
set(STENO_NEED_DICT_EMBED TRUE)
|
||||
endif()
|
||||
# ── v4 union split-section format ────────────────────
|
||||
# One compiler run emits BOTH half blobs (docs/FORMAT_V4.md §5).
|
||||
# Left (central): DISP, MEMBERSHIP, FP, CONFLICTS, VALIDX slice [0, k).
|
||||
# Right (peripheral): STRDIR, STRINGS, CONFLICTS, VALIDX slice [k, n).
|
||||
if(CONFIG_STENO_DICT_V4)
|
||||
|
||||
if(STENO_NEED_DICT_EMBED)
|
||||
find_package(Python3 REQUIRED COMPONENTS Interpreter)
|
||||
set(STENO_DICT_BIN ${CMAKE_CURRENT_BINARY_DIR}/steno_dict.bin)
|
||||
set(STENO_DICTS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/dicts)
|
||||
set(STENO_FETCH ${CMAKE_CURRENT_SOURCE_DIR}/tools/fetch_dict.py)
|
||||
if(NOT CONFIG_STENO_SPLIT_DICT)
|
||||
message(FATAL_ERROR
|
||||
"STENO_DICT_V4 requires STENO_SPLIT_DICT: the v4 format spans both "
|
||||
"keyboard halves (left = lookup structures, right = string table). "
|
||||
"Enable STENO_SPLIT_DICT, or disable STENO_DICT_V4 to fall back to "
|
||||
"the legacy MPHF format.")
|
||||
endif()
|
||||
|
||||
set(STENO_V4_LEFT_BIN ${CMAKE_CURRENT_BINARY_DIR}/steno_v4_left.bin)
|
||||
set(STENO_V4_RIGHT_BIN ${CMAKE_CURRENT_BINARY_DIR}/steno_v4_right.bin)
|
||||
|
||||
if(CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
|
||||
# Central: behavior engine + left half blob + BLE client
|
||||
target_sources(app PRIVATE
|
||||
src/behavior_steno.c
|
||||
src/output.c
|
||||
src/formatter.c
|
||||
src/undo.c
|
||||
src/dict_v4.c
|
||||
src/split_dict.c
|
||||
src/split_cache.c
|
||||
src/dict_embed.S
|
||||
)
|
||||
set(STENO_DICT_BIN ${STENO_V4_LEFT_BIN})
|
||||
else()
|
||||
# Peripheral: right half blob + GATT server
|
||||
target_sources(app PRIVATE
|
||||
src/dict_v4.c
|
||||
src/split_dict.c
|
||||
src/split_cache.c
|
||||
src/dict_embed.S
|
||||
)
|
||||
set(STENO_DICT_BIN ${STENO_V4_RIGHT_BIN})
|
||||
endif()
|
||||
|
||||
# The union format is always compiled from both source dicts;
|
||||
# --dicts controls which of them are included (dicts_mask).
|
||||
set(STENO_PLOVER_JSON ${STENO_DICTS_DIR}/plover-main.json)
|
||||
set(STENO_LAPWING_JSON ${STENO_DICTS_DIR}/lapwing.json)
|
||||
steno_fetch_dict(plover ${STENO_PLOVER_JSON})
|
||||
steno_fetch_dict(lapwing ${STENO_LAPWING_JSON})
|
||||
|
||||
# Single run produces both blobs. The compiler FAILS the build if
|
||||
# either half exceeds its budget — entries are never trimmed.
|
||||
add_custom_command(
|
||||
OUTPUT ${STENO_V4_LEFT_BIN} ${STENO_V4_RIGHT_BIN}
|
||||
COMMAND ${Python3_EXECUTABLE}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_v4.py
|
||||
--plover ${STENO_PLOVER_JSON} --lapwing ${STENO_LAPWING_JSON}
|
||||
--dicts ${STENO_DICTS_ARG}
|
||||
--out-dir ${CMAKE_CURRENT_BINARY_DIR}
|
||||
--left-size ${CONFIG_STENO_DICT_LEFT_MAX_SIZE}
|
||||
--right-size ${CONFIG_STENO_DICT_RIGHT_MAX_SIZE}
|
||||
DEPENDS
|
||||
${STENO_PLOVER_JSON} ${STENO_LAPWING_JSON}
|
||||
${STENO_PLOVER_JSON}.stamp ${STENO_LAPWING_JSON}.stamp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_v4.py
|
||||
COMMENT "Compiling steno v4 union dictionary (left + right half blobs)"
|
||||
)
|
||||
|
||||
add_custom_target(steno_dict_gen DEPENDS ${STENO_DICT_BIN})
|
||||
add_dependencies(app steno_dict_gen)
|
||||
|
||||
# ── Legacy MPHF format (STENO_DICT_V4=n fallback) ────
|
||||
else()
|
||||
|
||||
if(CONFIG_STENO_DICT_BOTH)
|
||||
message(FATAL_ERROR
|
||||
"STENO_DICT_BOTH requires STENO_DICT_V4: the legacy MPHF format "
|
||||
"holds a single dictionary. Select STENO_DICT_PLOVER or "
|
||||
"STENO_DICT_LAPWING, or re-enable STENO_DICT_V4.")
|
||||
endif()
|
||||
|
||||
if(CONFIG_STENO_DICT_PLOVER)
|
||||
set(STENO_DICT_NAME "plover")
|
||||
set(STENO_DICT_SRC ${STENO_DICTS_DIR}/plover-main.json)
|
||||
elseif(CONFIG_STENO_DICT_LAPWING)
|
||||
else()
|
||||
set(STENO_DICT_NAME "lapwing")
|
||||
set(STENO_DICT_SRC ${STENO_DICTS_DIR}/lapwing.json)
|
||||
else()
|
||||
set(STENO_DICT_SRC ${STENO_DICTS_DIR}/test.json)
|
||||
endif()
|
||||
|
||||
# Auto-download dict if needed (Plover/Lapwing only)
|
||||
if(DEFINED STENO_DICT_NAME AND NOT EXISTS ${STENO_DICT_SRC})
|
||||
message(STATUS "Steno: downloading ${STENO_DICT_NAME} dictionary...")
|
||||
execute_process(
|
||||
COMMAND ${Python3_EXECUTABLE} ${STENO_FETCH}
|
||||
${STENO_DICT_NAME} ${STENO_DICTS_DIR}
|
||||
RESULT_VARIABLE FETCH_RESULT
|
||||
)
|
||||
if(NOT FETCH_RESULT EQUAL 0)
|
||||
message(WARNING "Steno: dict download failed. Build may fail.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# Compile dict binary
|
||||
if(EXISTS ${STENO_DICT_SRC})
|
||||
# Fetch at build time if hash changed
|
||||
if(DEFINED STENO_DICT_NAME)
|
||||
add_custom_command(
|
||||
OUTPUT ${STENO_DICT_SRC}.stamp
|
||||
COMMAND ${Python3_EXECUTABLE} ${STENO_FETCH}
|
||||
${STENO_DICT_NAME} ${STENO_DICTS_DIR}
|
||||
COMMAND ${CMAKE_COMMAND} -E touch ${STENO_DICT_SRC}.stamp
|
||||
COMMENT "Checking ${STENO_DICT_NAME} dictionary for updates"
|
||||
if(CONFIG_STENO_SPLIT_DICT)
|
||||
if(CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
|
||||
# Central: behavior engine + local dict + BLE client
|
||||
target_sources(app PRIVATE
|
||||
src/behavior_steno.c
|
||||
src/output.c
|
||||
src/formatter.c
|
||||
src/undo.c
|
||||
src/dict_embed.S
|
||||
src/dict_mphf.c
|
||||
src/split_dict.c
|
||||
src/split_cache.c
|
||||
)
|
||||
else()
|
||||
# Peripheral: local dict + GATT server for central queries
|
||||
target_sources(app PRIVATE
|
||||
src/dict_embed.S
|
||||
src/dict_mphf.c
|
||||
src/split_dict.c
|
||||
src/split_cache.c
|
||||
)
|
||||
add_custom_target(steno_dict_fetch DEPENDS ${STENO_DICT_SRC}.stamp)
|
||||
endif()
|
||||
elseif(NOT CONFIG_ZMK_SPLIT OR CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
|
||||
target_sources(app PRIVATE
|
||||
src/behavior_steno.c
|
||||
src/output.c
|
||||
src/formatter.c
|
||||
src/undo.c
|
||||
src/dict_embed.S
|
||||
src/dict_mphf.c
|
||||
)
|
||||
endif()
|
||||
|
||||
# Dict embed needed on: both halves in split mode, else central /
|
||||
# non-split only.
|
||||
set(STENO_NEED_DICT_EMBED FALSE)
|
||||
if(CONFIG_STENO_SPLIT_DICT)
|
||||
set(STENO_NEED_DICT_EMBED TRUE)
|
||||
elseif(NOT CONFIG_ZMK_SPLIT OR CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
|
||||
set(STENO_NEED_DICT_EMBED TRUE)
|
||||
endif()
|
||||
|
||||
if(STENO_NEED_DICT_EMBED)
|
||||
set(STENO_DICT_BIN ${CMAKE_CURRENT_BINARY_DIR}/steno_dict.bin)
|
||||
steno_fetch_dict(${STENO_DICT_NAME} ${STENO_DICT_SRC})
|
||||
|
||||
if(CONFIG_STENO_SPLIT_DICT)
|
||||
# Split mode: compile the appropriate partition
|
||||
if(CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
|
||||
set(STENO_SPLIT_PART "left")
|
||||
set(STENO_SPLIT_SIZE ${CONFIG_STENO_DICT_LEFT_MAX_SIZE})
|
||||
|
|
@ -128,43 +199,39 @@ if(STENO_NEED_DICT_EMBED)
|
|||
--right-size ${CONFIG_STENO_DICT_RIGHT_MAX_SIZE}
|
||||
--max-size ${STENO_SPLIT_SIZE}
|
||||
--block-size 2048
|
||||
DEPENDS ${STENO_DICT_SRC}
|
||||
DEPENDS ${STENO_DICT_SRC} ${STENO_DICT_SRC}.stamp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_mphf.py
|
||||
COMMENT "Compiling steno dictionary (${STENO_SPLIT_PART} partition)"
|
||||
)
|
||||
elseif(CONFIG_STENO_DICT_MPHF)
|
||||
else()
|
||||
add_custom_command(
|
||||
OUTPUT ${STENO_DICT_BIN}
|
||||
COMMAND ${Python3_EXECUTABLE}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_mphf.py
|
||||
${STENO_DICT_SRC} ${STENO_DICT_BIN}
|
||||
--max-size ${CONFIG_STENO_DICT_MAX_SIZE}
|
||||
DEPENDS ${STENO_DICT_SRC}
|
||||
DEPENDS ${STENO_DICT_SRC} ${STENO_DICT_SRC}.stamp
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_mphf.py
|
||||
COMMENT "Compiling steno dictionary (MPHF)"
|
||||
)
|
||||
else()
|
||||
add_custom_command(
|
||||
OUTPUT ${STENO_DICT_BIN}
|
||||
COMMAND ${Python3_EXECUTABLE}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_simple.py
|
||||
${STENO_DICT_SRC} -o ${STENO_DICT_BIN}
|
||||
DEPENDS ${STENO_DICT_SRC}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_simple.py
|
||||
COMMENT "Compiling steno dictionary (simple)"
|
||||
)
|
||||
endif()
|
||||
|
||||
add_custom_target(steno_dict_gen DEPENDS ${STENO_DICT_BIN})
|
||||
if(TARGET steno_dict_fetch)
|
||||
add_dependencies(steno_dict_gen steno_dict_fetch)
|
||||
endif()
|
||||
add_dependencies(app steno_dict_gen)
|
||||
|
||||
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/steno_dict_path.h
|
||||
"#define STENO_DICT_BIN_PATH \"${STENO_DICT_BIN}\"\n")
|
||||
target_include_directories(app PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
|
||||
endif()
|
||||
|
||||
endif() # CONFIG_STENO_DICT_V4
|
||||
|
||||
# ── Dict embed plumbing ──────────────────────────────
|
||||
# dict_embed.S .incbin's the blob named by the generated header;
|
||||
# OBJECT_DEPENDS makes it re-assemble when the blob changes.
|
||||
if(STENO_DICT_BIN)
|
||||
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/steno_dict_path.h
|
||||
"#define STENO_DICT_BIN_PATH \"${STENO_DICT_BIN}\"\n")
|
||||
target_include_directories(app PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
|
||||
set_source_files_properties(${CMAKE_CURRENT_SOURCE_DIR}/src/dict_embed.S
|
||||
TARGET_DIRECTORY app
|
||||
PROPERTIES OBJECT_DEPENDS ${STENO_DICT_BIN})
|
||||
endif()
|
||||
|
||||
endif() # CONFIG_STENO_ENGINE
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue