afiqzudinhadi/zmk-steno-engine
1
0
Fork 0
Code Issues 3 Pull requests Projects Releases Packages Wiki Activity Actions

Steno engine with BLE split-storage architecture

Browse source 
Full steno engine: chord detection, MPHF/trie dict lookup, Plover
formatter (18 commands), undo (ring buffer), HID + Unicode output.

Split-storage specific:
- BLE GATT service for dict queries (query/prefix/batch)
- LRU cache on central side (CONFIG_STENO_SPLIT_CACHE_SIZE)
- Dict embedded on peripheral, engine runs on central
- 3-way dispatch: split_dict / MPHF / simple trie

Build system: auto-fetch Plover dict, MPHF compiler with block
compression (~56K entries in 420KB), CMake integration.
This commit is contained in:
afiqzudinhadi 2026-07-02 02:06:37 +08:00
parent 234bc8d731
commit 2af45eafce
36 changed files with 5969 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

6
.gitignore vendored
View file

@ -0,0 +1,6 @@
__pycache__/
*.pyc
dicts/plover-main.json
dicts/lapwing.json
dicts/*.sha256
tests/build/

145
CMakeLists.txt Normal file
View file

@ -0,0 +1,145 @@
# Copyright (c) 2024 Afiq Zudin Hadi
# SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
if(CONFIG_STENO_ENGINE)
target_include_directories(app PRIVATE
include
${CMAKE_CURRENT_SOURCE_DIR}/src
)
# ── Split-dict mode ──────────────────────────────────
# Central: behavior engine + BLE dict client
# Peripheral: dict embed + lookup engine + GATT server
if(CONFIG_STENO_SPLIT_DICT)
if(CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
# Central side: behavior engine queries peripheral over BLE
target_sources(app PRIVATE
src/behavior_steno.c
src/output.c
src/formatter.c
src/undo.c
src/split_dict.c
src/split_cache.c
)
else()
# Peripheral side: dict embedded here, serves GATT queries
target_sources(app PRIVATE
src/dict_embed.S
)
if(CONFIG_STENO_DICT_MPHF)
target_sources(app PRIVATE src/dict_mphf.c)
else()
target_sources(app PRIVATE src/trie.c)
endif()
endif()
# ── Non-split mode ───────────────────────────────────
# Everything on one board (or central-only without split)
else()
if(NOT CONFIG_ZMK_SPLIT OR CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
target_sources(app PRIVATE
src/behavior_steno.c
src/output.c
src/formatter.c
src/undo.c
src/dict_embed.S
)
if(CONFIG_STENO_DICT_MPHF)
target_sources(app PRIVATE src/dict_mphf.c)
else()
target_sources(app PRIVATE src/trie.c)
endif()
endif()
endif()
# ── Dictionary compilation ───────────────────────────
# Build dict binary when we embed it (non-split, or split peripheral)
set(STENO_NEED_DICT_EMBED FALSE)
if(CONFIG_STENO_SPLIT_DICT)
if(NOT CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
set(STENO_NEED_DICT_EMBED TRUE)
endif()
elseif(NOT CONFIG_ZMK_SPLIT OR CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
set(STENO_NEED_DICT_EMBED TRUE)
endif()
if(STENO_NEED_DICT_EMBED)
find_package(Python3 REQUIRED COMPONENTS Interpreter)
set(STENO_DICT_BIN ${CMAKE_CURRENT_BINARY_DIR}/steno_dict.bin)
set(STENO_DICTS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/dicts)
set(STENO_FETCH ${CMAKE_CURRENT_SOURCE_DIR}/tools/fetch_dict.py)
if(CONFIG_STENO_DICT_PLOVER)
set(STENO_DICT_NAME "plover")
set(STENO_DICT_SRC ${STENO_DICTS_DIR}/plover-main.json)
elseif(CONFIG_STENO_DICT_LAPWING)
set(STENO_DICT_NAME "lapwing")
set(STENO_DICT_SRC ${STENO_DICTS_DIR}/lapwing.json)
else()
set(STENO_DICT_SRC ${STENO_DICTS_DIR}/test.json)
endif()
# Auto-download dict if needed (Plover/Lapwing only)
if(DEFINED STENO_DICT_NAME AND NOT EXISTS ${STENO_DICT_SRC})
message(STATUS "Steno: downloading ${STENO_DICT_NAME} dictionary...")
execute_process(
COMMAND ${Python3_EXECUTABLE} ${STENO_FETCH}
${STENO_DICT_NAME} ${STENO_DICTS_DIR}
RESULT_VARIABLE FETCH_RESULT
)
if(NOT FETCH_RESULT EQUAL 0)
message(WARNING "Steno: dict download failed. Build may fail.")
endif()
endif()
# Compile dict binary
if(EXISTS ${STENO_DICT_SRC})
if(CONFIG_STENO_DICT_MPHF)
# Fetch at build time if hash changed
if(DEFINED STENO_DICT_NAME)
add_custom_command(
OUTPUT ${STENO_DICT_SRC}.stamp
COMMAND ${Python3_EXECUTABLE} ${STENO_FETCH}
${STENO_DICT_NAME} ${STENO_DICTS_DIR}
COMMAND ${CMAKE_COMMAND} -E touch ${STENO_DICT_SRC}.stamp
COMMENT "Checking ${STENO_DICT_NAME} dictionary for updates"
)
add_custom_target(steno_dict_fetch DEPENDS ${STENO_DICT_SRC}.stamp)
endif()
add_custom_command(
OUTPUT ${STENO_DICT_BIN}
COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_mphf.py
${STENO_DICT_SRC} ${STENO_DICT_BIN}
--max-size ${CONFIG_STENO_DICT_MAX_SIZE}
DEPENDS ${STENO_DICT_SRC}
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_mphf.py
COMMENT "Compiling steno dictionary (MPHF)"
)
else()
add_custom_command(
OUTPUT ${STENO_DICT_BIN}
COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_simple.py
${STENO_DICT_SRC} -o ${STENO_DICT_BIN}
DEPENDS ${STENO_DICT_SRC}
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_simple.py
COMMENT "Compiling steno dictionary (simple)"
)
endif()
add_custom_target(steno_dict_gen DEPENDS ${STENO_DICT_BIN})
if(TARGET steno_dict_fetch)
add_dependencies(steno_dict_gen steno_dict_fetch)
endif()
add_dependencies(app steno_dict_gen)
set_property(SOURCE src/dict_embed.S APPEND PROPERTY
COMPILE_DEFINITIONS STENO_DICT_BIN_PATH="${STENO_DICT_BIN}")
endif()
endif()
endif() # CONFIG_STENO_ENGINE

127
Kconfig Normal file
View file

@ -0,0 +1,127 @@
# Copyright (c) 2024 Afiq Zudin Hadi
# SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
menuconfig STENO_ENGINE
bool "Steno Engine"
default n
help
Enable the stenography engine for ZMK.
if STENO_ENGINE
# ──────────────────────────────────────────────
# Dictionary selection
# ──────────────────────────────────────────────
choice STENO_DICT
prompt "Steno dictionary"
default STENO_DICT_PLOVER
config STENO_DICT_PLOVER
bool "Plover main dictionary (MPHF compressed)"
help
Use Plover main.json via MPHF compression (~44K entries in 453KB).
config STENO_DICT_LAPWING
bool "Lapwing dictionary (MPHF compressed)"
config STENO_DICT_TEST
bool "Test dictionary (46 entries, simple format)"
help
Small built-in test dictionary for development.
endchoice
config STENO_DICT_MPHF
bool
default y if STENO_DICT_PLOVER || STENO_DICT_LAPWING
help
Use MPHF (minimal perfect hash) dictionary format.
Auto-selected for Plover/Lapwing dicts.
# ──────────────────────────────────────────────
# Unicode output mode
# ──────────────────────────────────────────────
choice STENO_UNICODE_MODE
prompt "Unicode output mode"
default STENO_UNICODE_MODE_NONE
config STENO_UNICODE_MODE_NONE
bool "None (HID only)"
config STENO_UNICODE_MODE_LINUX
bool "Linux (IBus)"
config STENO_UNICODE_MODE_MACOS
bool "macOS"
config STENO_UNICODE_MODE_WINC
bool "Windows (WinCompose)"
endchoice
config STENO_HISTORY_SIZE
int "Stroke history size (undo depth)"
default 100
range 10 500
config STENO_KEY_DELAY_MS
int "Key output delay (ms)"
default 0
range 0 50
config STENO_MULTI_STROKE_TIMEOUT_MS
int "Multi-stroke timeout (ms)"
default 500
range 100 5000
config STENO_DICT_MAX_SIZE
int "Max dictionary binary size (bytes)"
default 473088
help
Max compiled dict size. 473088 = 462KB (left half flash budget).
The MPHF compiler auto-trims to fit.
# ──────────────────────────────────────────────
# Split keyboard dictionary storage
# ──────────────────────────────────────────────
menuconfig STENO_SPLIT_DICT
bool "Split dictionary storage on peripheral"
default n
help
Store the steno dictionary on the peripheral half and
perform lookups over BLE. This frees flash on the central
side at the cost of added lookup latency.
if STENO_SPLIT_DICT
config STENO_SPLIT_CACHE_SIZE
int "LRU cache entries on central side"
default 64
range 16 512
help
Number of recently looked-up dictionary entries cached
on the central side to avoid repeated BLE round-trips.
config STENO_SPLIT_PREFETCH
bool "Prefetch common follow-up strokes"
default y
help
After a successful lookup, speculatively prefetch dictionary
entries for statistically common follow-up strokes. Reduces
perceived latency for multi-stroke words.
config STENO_SPLIT_TIMEOUT_MS
int "BLE lookup timeout (ms)"
default 50
range 10 500
help
Maximum time in milliseconds to wait for a dictionary
lookup response from the peripheral over BLE before
falling back to a cache miss / no-op.
endif # STENO_SPLIT_DICT
endif # STENO_ENGINE

131
LICENSE Normal file
View file

@ -0,0 +1,131 @@
# PolyForm Noncommercial License 1.0.0
<https://polyformproject.org/licenses/noncommercial/1.0.0>
## Acceptance
In order to get any license under these terms, you must agree
to them as both strict obligations and conditions to all
your licenses.
## Copyright License
The licensor grants you a copyright license for the
software to do everything you might do with the software
that would otherwise infringe the licensor's copyright
in it for any permitted purpose. However, you may
only distribute the software according to [Distribution
License](#distribution-license) and make changes or new works
based on the software according to [Changes and New Works
License](#changes-and-new-works-license).
## Distribution License
The licensor grants you an additional copyright license
to distribute copies of the software. Your license
to distribute covers distributing the software with
changes and new works permitted by [Changes and New Works
License](#changes-and-new-works-license).
## Notices
You must ensure that anyone who gets a copy of any part of
the software from you also gets a copy of these terms or the
URL for them above, as well as copies of any plain-text lines
beginning with `Required Notice:` that the licensor provided
with the software. For example:
> Required Notice: Copyright Yoyodyne, Inc. (http://example.com)
## Changes and New Works License
The licensor grants you an additional copyright license to
make changes and new works based on the software for any
permitted purpose.
## Patent License
The licensor grants you a patent license for the software that
covers patent claims the licensor can license, or becomes able
to license, that you would infringe by using the software.
## Noncommercial Purposes
Any noncommercial purpose is a permitted purpose.
## Personal Uses
Personal use for research, experiment, and testing for
the benefit of public knowledge, personal study, private
entertainment, hobby projects, amateur pursuits, or religious
observance, without any anticipated commercial application,
is use for a permitted purpose.
## Noncommercial Organizations
Use by any charitable organization, educational institution,
public research organization, public safety or health
organization, environmental protection organization,
or government institution is use for a permitted purpose
regardless of the source of funding or obligations resulting
from the funding.
## Fair Use
You may have "fair use" rights for the software under the
law. These terms do not limit them.
## No Other Rights
These terms do not allow you to sublicense or transfer any of
your licenses to anyone else, or prevent the licensor from
granting licenses to anyone else. These terms do not imply
any other licenses.
## Patent Defense
If you make any written claim that the software infringes or
contributes to infringement of any patent, your patent license
for the software granted under these terms ends immediately. If
your company makes such a claim, your patent license ends
immediately for work on behalf of your company.
## Violations
The first time you are notified in writing that you have
violated any of these terms, or done anything with the software
not covered by your licenses, your licenses can nonetheless
continue if you come into full compliance with these terms,
and take practical steps to correct past violations, within
32 days of receiving notice. Otherwise, all your licenses
end immediately.
## No Liability
***As far as the law allows, the software comes as is, without
any warranty or condition, and the licensor will not be liable
to you for any damages arising out of these terms or the use
or nature of the software, under any kind of legal claim.***
## Definitions
The **licensor** is the individual or entity offering these
terms, and the **software** is the software the licensor makes
available under these terms.
**You** refers to the individual or entity agreeing to these
terms.
**Your company** is any legal entity, sole proprietorship,
or other kind of organization that you work for, plus all
organizations that have control over, are under the control of,
or are under common control with that organization. **Control**
means ownership of substantially all the assets of an entity,
or the power to direct its management and policies by vote,
contract, or otherwise. Control can be direct or indirect.
**Your licenses** are all the licenses granted to you for the
software under these terms.
**Use** means anything you do with the software requiring one
of your licenses.

30
README.md Normal file
View file

@ -0,0 +1,30 @@
# zmk-steno-engine
Clean-room steno engine for ZMK keyboards. Split-storage architecture: full dictionary on peripheral (right) half, queried over BLE from central (left) half.
**Status:** Early development — not yet functional.
## Architecture
ZMK splits run two nRF52840 halves connected via BLE. The right (peripheral) half has ~533KB free flash — enough for a full steno dictionary. The left (central) half runs the steno engine.
```
┌─────────────┐ BLE GATT query ┌──────────────┐
│ Left half │ ───────────────► │ Right half │
│ (central) │ ◄─────────────── │ (peripheral) │
│ │ translation │ │
│ Steno engine│ │ Dictionary │
│ LRU cache │ │ (~533KB) │
└─────────────┘ └──────────────┘
```
**Key design decisions:**
- Dictionary stored on peripheral → no flash pressure on central
- BLE GATT custom service for stroke→translation queries
- LRU cache on central side → reduces BLE round-trips for common words
- Orthographic rules run on central after dictionary lookup
## License
[PolyForm Noncommercial 1.0.0](LICENSE)

48
dicts/test.json Normal file
View file

@ -0,0 +1,48 @@
{
"S": "is",
"T": "it",
"-T": "the",
"K": "can",
"W": "with",
"H": "had",
"R": "are",
"A": "a",
"AOEU": "I",
"TKPW": "go",
"HRAO": "hello",
"WORBGD": "world",
"KAT": "cat",
"TKOG": "dog",
"HOUS": "house",
"TPEUR": "fire",
"WAURT": "water",
"TPOOD": "food",
"PWO*BG": "book",
"KO": "co",
"KO/PHAOURD": "computer",
"KO/PHAOURDZ": "computers",
"TEFT": "test",
"TEFTD": "tested",
"TEFTG": "testing",
"STE/TPHO": "steno",
"STE/TPHOG": "stenography",
"KPA": "{}{-|}",
"S-P": "{^ ^}",
"TP-PL": "{.}",
"KW-BG": "{,}",
"H-F": "{?}",
"SKHRAPL": "{!}",
"*": "{*}",
"TPHO": "no",
"KWRA": "yes",
"THA": "that",
"PWUT": "but",
"TPHOT": "not",
"SREU": "have",
"WHA": "what",
"HEU": "he",
"SHEU": "she",
"THE": "they",
"WE": "we",
"KWREU": "I"
}

8
dts/behaviors/steno_engine.dtsi Normal file
View file

@ -0,0 +1,8 @@
/ {
behaviors {
/omit-if-no-ref/ steno: steno_engine {
compatible = "zmk,behavior-steno-engine";
#binding-cells = <1>;
};
};
};

3
dts/bindings/behaviors/zmk,behavior-steno-engine.yaml Normal file
View file

@ -0,0 +1,3 @@
description: Steno engine behavior — maps physical keys to steno positions
compatible: "zmk,behavior-steno-engine"
include: one_param.yaml

53
include/dt-bindings/zmk/steno_keys.h Normal file
View file

@ -0,0 +1,53 @@
/*
* Copyright (c) 2024 Afiq Zudin Hadi
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*
* Standard steno key layout. Each key = bit position in uint32_t chord.
* Bit assignments match the compiler's STENO_KEYS bitmask exactly.
*
* Layout (standard steno order):
* S T K P W H R A O * E U F R P B L G T S D Z #
* 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
*/
#ifndef DT_BINDINGS_ZMK_STENO_KEYS_H_
#define DT_BINDINGS_ZMK_STENO_KEYS_H_
/* Left hand consonants */
#define STENO_SL 0 /* S- (left) */
#define STENO_TL 1 /* T- */
#define STENO_KL 2 /* K- */
#define STENO_PL 3 /* P- */
#define STENO_WL 4 /* W- */
#define STENO_HL 5 /* H- */
#define STENO_RL 6 /* R- (left) */
/* Vowels */
#define STENO_A 7 /* A */
#define STENO_O 8 /* O */
/* Center */
#define STENO_STAR 9 /* * */
/* Vowels (right) */
#define STENO_E 10 /* E */
#define STENO_U 11 /* U */
/* Right hand consonants */
#define STENO_FR 12 /* -F */
#define STENO_RR 13 /* -R (right) */
#define STENO_PR 14 /* -P (right) */
#define STENO_BR 15 /* -B */
#define STENO_LR 16 /* -L */
#define STENO_GR 17 /* -G */
#define STENO_TR 18 /* -T (right) */
#define STENO_SR 19 /* -S (right) */
#define STENO_DR 20 /* -D */
#define STENO_ZR 21 /* -Z */
/* Number bar */
#define STENO_NUM 22 /* # */
#define STENO_KEY_COUNT 23
#endif /* DT_BINDINGS_ZMK_STENO_KEYS_H_ */

270
src/behavior_steno.c Normal file
View file

@ -0,0 +1,270 @@
/*
* Copyright (c) 2024 Afiq Zudin Hadi
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*/
#define DT_DRV_COMPAT zmk_behavior_steno_engine
#include <zephyr/device.h>
#include <zephyr/kernel.h>
#include <zephyr/logging/log.h>
#include <drivers/behavior.h>
#include <zmk/behavior.h>
#include "output.h"
#include "undo.h"
#include "formatter.h"
#if IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
#include "split_dict.h"
#elif IS_ENABLED(CONFIG_STENO_DICT_MPHF)
#include "dict_mphf.h"
#else
#include "trie.h"
#endif
LOG_MODULE_DECLARE(zmk, CONFIG_ZMK_LOG_LEVEL);
#if !IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
extern const uint8_t _steno_dict_start[];
extern const uint8_t _steno_dict_end[];
#endif
#if IS_ENABLED(CONFIG_STENO_DICT_MPHF) && !IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
static struct dict_mphf mphf_dict;
#endif
#define STENO_MAX_MULTI 8
#define STENO_MULTI_TIMEOUT_MS CONFIG_STENO_MULTI_STROKE_TIMEOUT_MS
struct steno_state {
uint32_t current_chord;
uint8_t keys_held;
uint32_t pending_strokes[STENO_MAX_MULTI];
uint8_t stroke_count;
struct k_work_delayable multi_timeout;
};
static struct steno_state state;
static struct steno_fmt_state fmt_state;
static struct stroke_history undo_history;
static bool dict_ready;
static void flush_strokes(void);
static void multi_timeout_handler(struct k_work *work);
static const char *do_lookup(const uint32_t *strokes, uint8_t count)
{
#if IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
static char split_buf[128];
int ret = split_dict_lookup(strokes, count, split_buf, sizeof(split_buf));
return (ret > 0) ? split_buf : NULL;
#elif IS_ENABLED(CONFIG_STENO_DICT_MPHF)
return dict_mphf_lookup(&mphf_dict, strokes, count);
#else
return steno_trie_lookup(strokes, count);
#endif
}
static bool do_has_prefix(const uint32_t *strokes, uint8_t count)
{
#if IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
return split_dict_has_prefix(strokes, count);
#elif IS_ENABLED(CONFIG_STENO_DICT_MPHF)
return (count == 1) ? dict_mphf_has_prefix(&mphf_dict, strokes[0]) : false;
#else
return steno_trie_has_prefix(strokes, count);
#endif
}
static void emit_formatted(const char *translation,
const uint32_t *strokes, uint8_t stroke_count)
{
struct steno_fmt_result result;
steno_fmt_process(&fmt_state, translation, &result);
if (result.backspaces > 0) {
steno_output_backspace(result.backspaces);
}
if (result.len > 0) {
steno_output_send(result.text, result.len);
}
if (!result.is_command_only) {
steno_undo_push(&undo_history, strokes, stroke_count,
result.len + result.backspaces, 0, 0);
}
}
static void process_chord(void)
{
if (state.current_chord == 0) {
return;
}
/* Star-only stroke (bit 9) → undo */
if (state.current_chord == (1U << 9)) {
state.current_chord = 0;
struct stroke_history_entry *ue = steno_undo_pop(&undo_history);
if (ue) {
steno_output_backspace(ue->output_len);
}
return;
}
k_work_cancel_delayable(&state.multi_timeout);
if (state.stroke_count < STENO_MAX_MULTI) {
state.pending_strokes[state.stroke_count] = state.current_chord;
state.stroke_count++;
} else {
flush_strokes();
state.pending_strokes[0] = state.current_chord;
state.stroke_count = 1;
}
state.current_chord = 0;
if (!dict_ready) {
flush_strokes();
return;
}
const char *translation = do_lookup(state.pending_strokes, state.stroke_count);
if (translation) {
emit_formatted(translation, state.pending_strokes, state.stroke_count);
state.stroke_count = 0;
return;
}
if (do_has_prefix(state.pending_strokes, state.stroke_count)) {
k_work_schedule(&state.multi_timeout,
K_MSEC(STENO_MULTI_TIMEOUT_MS));
return;
}
if (state.stroke_count > 1) {
uint32_t last = state.pending_strokes[state.stroke_count - 1];
state.stroke_count--;
const char *partial = do_lookup(state.pending_strokes, state.stroke_count);
if (partial) {
emit_formatted(partial, state.pending_strokes, state.stroke_count);
}
state.pending_strokes[0] = last;
state.stroke_count = 1;
const char *rest = do_lookup(&last, 1);
if (rest) {
emit_formatted(rest, &last, 1);
state.stroke_count = 0;
}
return;
}
flush_strokes();
}
static void flush_strokes(void)
{
LOG_DBG("Flushing %u untranslated strokes", state.stroke_count);
state.stroke_count = 0;
}
static void multi_timeout_handler(struct k_work *work)
{
ARG_UNUSED(work);
if (state.stroke_count == 0) {
return;
}
const char *translation = do_lookup(state.pending_strokes, state.stroke_count);
if (translation) {
emit_formatted(translation, state.pending_strokes, state.stroke_count);
}
state.stroke_count = 0;
}
static int on_steno_binding_pressed(struct zmk_behavior_binding *binding,
struct zmk_behavior_binding_event event)
{
uint32_t key_index = binding->param1;
if (key_index > 35) {
return -EINVAL;
}
state.current_chord |= (1U << key_index);
state.keys_held++;
LOG_DBG("Key %u pressed, chord=0x%06X held=%u",
key_index, state.current_chord, state.keys_held);
return ZMK_BEHAVIOR_OPAQUE;
}
static int on_steno_binding_released(struct zmk_behavior_binding *binding,
struct zmk_behavior_binding_event event)
{
if (state.keys_held > 0) {
state.keys_held--;
}
if (state.keys_held == 0 && state.current_chord != 0) {
process_chord();
}
return ZMK_BEHAVIOR_OPAQUE;
}
static int behavior_steno_init(const struct device *dev)
{
ARG_UNUSED(dev);
state.current_chord = 0;
state.keys_held = 0;
state.stroke_count = 0;
steno_fmt_init(&fmt_state);
steno_undo_init(&undo_history);
k_work_init_delayable(&state.multi_timeout, multi_timeout_handler);
#if IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
split_dict_init();
dict_ready = true;
#else
size_t dict_size = _steno_dict_end - _steno_dict_start;
if (dict_size > 4) {
int ret;
#if IS_ENABLED(CONFIG_STENO_DICT_MPHF)
ret = dict_mphf_init(&mphf_dict, _steno_dict_start, dict_size);
#else
ret = steno_trie_init(_steno_dict_start, dict_size);
#endif
if (ret == 0) {
dict_ready = true;
LOG_INF("Steno dict loaded (%u bytes)", (unsigned)dict_size);
} else {
LOG_ERR("Steno dict init failed: %d", ret);
}
} else {
LOG_WRN("No steno dict embedded");
}
#endif
LOG_INF("Steno engine initialized");
return 0;
}
static const struct behavior_driver_api steno_driver_api = {
.locality = BEHAVIOR_LOCALITY_CENTRAL,
.binding_pressed = on_steno_binding_pressed,
.binding_released = on_steno_binding_released,
};
#define STENO_INST(n) \
BEHAVIOR_DT_INST_DEFINE(n, \
behavior_steno_init, \
NULL, \
NULL, \
NULL, \
POST_KERNEL, \
CONFIG_KERNEL_INIT_PRIORITY_DEFAULT, \
&steno_driver_api);
DT_INST_FOREACH_STATUS_OKAY(STENO_INST)

12
src/dict_embed.S Normal file
View file

@ -0,0 +1,12 @@
.section .rodata.steno_dict, "a", %progbits
.global _steno_dict_start
.global _steno_dict_end
.balign 4
_steno_dict_start:
#ifdef STENO_DICT_BIN_PATH
.incbin STENO_DICT_BIN_PATH
#else
.byte 0x00, 0x00, 0x00, 0x00
#endif
_steno_dict_end:

231
src/dict_mphf.c Normal file
View file

@ -0,0 +1,231 @@
/**
* MPHF dictionary lookup engine — implementation.
*
* All data is read directly from flash. No heap allocation.
* Bit-packed fields read via inline bit extraction.
*
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*/
#include "dict_mphf.h"
#include <string.h>
/* ─── FNV-1a 32-bit hash ─── */
static uint32_t fnv1a_32(const uint8_t *data, size_t len)
{
uint32_t h = 0x811c9dc5u;
for (size_t i = 0; i < len; i++) {
h ^= data[i];
h *= 0x01000193u;
}
return h;
}
/**
* Hash key bytes with a seed (prepend seed as LE u32).
* Equivalent to: fnv1a_32(pack('<I', seed) + key_bytes)
*/
static uint32_t hash_key(const uint8_t *key, size_t key_len, uint32_t seed)
{
uint32_t h = 0x811c9dc5u;
/* Hash the seed bytes first (little-endian u32) */
uint8_t seed_bytes[4];
seed_bytes[0] = (uint8_t)(seed);
seed_bytes[1] = (uint8_t)(seed >> 8);
seed_bytes[2] = (uint8_t)(seed >> 16);
seed_bytes[3] = (uint8_t)(seed >> 24);
for (int i = 0; i < 4; i++) {
h ^= seed_bytes[i];
h *= 0x01000193u;
}
/* Then hash the key bytes */
for (size_t i = 0; i < key_len; i++) {
h ^= key[i];
h *= 0x01000193u;
}
return h;
}
/* ─── Bit-packed field reading ─── */
/**
* Read n_bits from a bit-packed array starting at bit position bit_pos.
* Bits are packed LSB-first within each byte.
*/
static uint32_t read_bits(const uint8_t *data, uint32_t bit_pos, uint8_t n_bits)
{
uint32_t value = 0;
for (uint8_t i = 0; i < n_bits; i++) {
uint32_t byte_idx = (bit_pos + i) / 8;
uint8_t bit_idx = (bit_pos + i) % 8;
if (data[byte_idx] & (1u << bit_idx)) {
value |= (1u << i);
}
}
return value;
}
/* ─── Alignment helper ─── */
static inline uint32_t align4(uint32_t n)
{
return (n + 3u) & ~3u;
}
/* ─── Init ─── */
int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len)
{
if (!dict || !data) {
return -1;
}
if (len < sizeof(struct dict_mphf_header)) {
return -2;
}
const struct dict_mphf_header *hdr = (const struct dict_mphf_header *)data;
if (hdr->magic != DICT_MPHF_MAGIC) {
return -3;
}
if (hdr->version != DICT_MPHF_VERSION) {
return -4;
}
dict->header = hdr;
const uint8_t *base = (const uint8_t *)data;
uint32_t offset = sizeof(struct dict_mphf_header);
/* Displacements section */
dict->displacements = base + offset;
uint32_t disp_bits_total = (uint32_t)hdr->bucket_count * hdr->disp_bits;
dict->disp_section_len = align4((disp_bits_total + 7) / 8);
offset += dict->disp_section_len;
/* Values section */
dict->values = base + offset;
uint32_t val_bits_total = (uint32_t)hdr->entry_count * hdr->value_bits;
dict->val_section_len = align4((val_bits_total + 7) / 8);
offset += dict->val_section_len;
/* Fingerprints section */
dict->fingerprints = base + offset;
dict->fp_section_len = align4(hdr->entry_count);
offset += dict->fp_section_len;
/* String offsets section (u24 LE, 3 bytes each) */
dict->string_offsets = base + offset;
offset += hdr->unique_count * 3;
/* String data section */
dict->string_data = (const char *)(base + offset);
/*
* To find string_data length, scan for end of last string.
* But we don't strictly need it for lookups — strings are
* null-terminated and we just follow offsets.
*
* For prefix_table, we need to know where string_data ends.
* Compute from total file size minus prefix table size.
*/
/* Prefix table is at the end of the file */
uint32_t prefix_bytes = (uint32_t)hdr->prefix_count * 4;
if (len >= prefix_bytes) {
dict->prefix_table = (const uint32_t *)(base + len - prefix_bytes);
} else {
dict->prefix_table = NULL;
}
return 0;
}
/* ─── Lookup ─── */
const char *dict_mphf_lookup(const struct dict_mphf *dict,
const uint32_t *strokes, uint8_t count)
{
if (!dict || !dict->header || !strokes || count == 0) {
return NULL;
}
const struct dict_mphf_header *hdr = dict->header;
/* Build key bytes: each stroke as LE u32, concatenated */
uint8_t key_buf[32]; /* max 8 strokes × 4 bytes */
size_t key_len = (size_t)count * 4;
if (key_len > sizeof(key_buf)) {
return NULL;
}
for (uint8_t i = 0; i < count; i++) {
key_buf[i * 4 + 0] = (uint8_t)(strokes[i]);
key_buf[i * 4 + 1] = (uint8_t)(strokes[i] >> 8);
key_buf[i * 4 + 2] = (uint8_t)(strokes[i] >> 16);
key_buf[i * 4 + 3] = (uint8_t)(strokes[i] >> 24);
}
/* MPHF lookup */
uint32_t bucket = hash_key(key_buf, key_len, 0) % hdr->bucket_count;
uint32_t d = read_bits(dict->displacements,
bucket * (uint32_t)hdr->disp_bits,
hdr->disp_bits);
uint32_t slot = hash_key(key_buf, key_len, d + 1) % hdr->entry_count;
/* Fingerprint check */
uint8_t expected_fp = (uint8_t)(fnv1a_32(key_buf, key_len) & 0xFF);
if (dict->fingerprints[slot] != expected_fp) {
return NULL;
}
/* Read value ID from bit-packed array */
uint32_t val_id = read_bits(dict->values,
slot * (uint32_t)hdr->value_bits,
hdr->value_bits);
if (val_id >= hdr->unique_count) {
return NULL;
}
/* Resolve string (u24 LE offset) */
const uint8_t *off_ptr = dict->string_offsets + val_id * 3;
uint32_t str_offset = (uint32_t)off_ptr[0]
| ((uint32_t)off_ptr[1] << 8)
| ((uint32_t)off_ptr[2] << 16);
return dict->string_data + str_offset;
}
/* ─── has_prefix ─── */
bool dict_mphf_has_prefix(const struct dict_mphf *dict, uint32_t stroke)
{
if (!dict || !dict->header || !dict->prefix_table ||
dict->header->prefix_count == 0) {
return false;
}
/* Binary search in sorted prefix table */
uint32_t lo = 0;
uint32_t hi = dict->header->prefix_count;
while (lo < hi) {
uint32_t mid = lo + (hi - lo) / 2;
uint32_t val = dict->prefix_table[mid];
if (val == stroke) {
return true;
} else if (val < stroke) {
lo = mid + 1;
} else {
hi = mid;
}
}
return false;
}

115
src/dict_mphf.h Normal file
View file

@ -0,0 +1,115 @@
/**
* MPHF (Minimal Perfect Hash Function) dictionary lookup engine.
*
* Binary format v2: CHD MPHF + bit-packed displacements/values +
* fingerprinted verification + deduped string table.
*
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*/
#ifndef DICT_MPHF_H
#define DICT_MPHF_H
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
#define DICT_MPHF_MAGIC 0x4F4E5453 /* "STNO" */
#define DICT_MPHF_VERSION 2
/**
* On-flash header (32 bytes). All fields little-endian.
*
* Layout:
* [0..3] magic u32
* [4..5] version u16
* [6..7] flags u16
* [8..11] entry_count u32
* [12..15] bucket_count u32
* [16..19] unique_count u32
* [20] value_bits u8
* [21] disp_bits u8
* [22..23] prefix_count u16
* [24..27] reserved0 u32
* [28..31] reserved1 u32
*/
struct dict_mphf_header {
uint32_t magic;
uint16_t version;
uint16_t flags;
uint32_t entry_count;
uint32_t bucket_count;
uint32_t unique_count;
uint8_t value_bits;
uint8_t disp_bits;
uint16_t prefix_count;
uint32_t reserved0;
uint32_t reserved1;
} __attribute__((packed));
_Static_assert(sizeof(struct dict_mphf_header) == 32, "header must be 32 bytes");
/**
* Runtime dictionary handle. Points into flash-resident data.
* All pointers are into the compiled binary blob — no heap allocation.
*/
struct dict_mphf {
const struct dict_mphf_header *header;
const uint8_t *displacements; /* bit-packed, bucket_count entries */
const uint8_t *values; /* bit-packed, entry_count entries */
const uint8_t *fingerprints; /* 1 byte per entry */
const uint8_t *string_offsets; /* unique_count offsets (u24 LE) */
const char *string_data; /* null-terminated UTF-8 strings */
const uint32_t *prefix_table; /* sorted first-strokes, prefix_count entries */
uint32_t disp_section_len;
uint32_t val_section_len;
uint32_t fp_section_len;
};
/**
* Initialize dictionary handle from a compiled binary blob.
*
* @param dict Handle to initialize
* @param data Pointer to compiled binary (typically flash-mapped)
* @param len Length of binary in bytes
* @return 0 on success, negative on error
* -1: NULL pointer
* -2: too short
* -3: bad magic
* -4: version mismatch
*/
int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len);
/**
* Look up a stroke sequence in the dictionary.
*
* @param dict Initialized dictionary handle
* @param strokes Array of stroke values (each is a 23-bit steno chord)
* @param count Number of strokes in sequence
* @return Pointer to null-terminated translation string in flash,
* or NULL if not found / fingerprint mismatch
*/
const char *dict_mphf_lookup(const struct dict_mphf *dict,
const uint32_t *strokes, uint8_t count);
/**
* Check if a stroke could be the prefix of a multi-stroke entry.
*
* Binary search on sorted prefix table of first-strokes from
* multi-stroke entries.
*
* @param dict Initialized dictionary handle
* @param stroke Single stroke value to check
* @return true if stroke appears as first stroke of any multi-stroke entry
*/
bool dict_mphf_has_prefix(const struct dict_mphf *dict, uint32_t stroke);
/**
* Get number of entries in the dictionary.
*/
static inline uint32_t dict_mphf_count(const struct dict_mphf *dict)
{
return dict->header->entry_count;
}
#endif /* DICT_MPHF_H */

335
src/formatter.c Normal file
View file

@ -0,0 +1,335 @@
/*
* Copyright (c) 2024 Afiq Zudin Hadi
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*
* Plover formatting engine — parses translation strings containing
* {commands} and produces clean output with spacing/capitalization.
* No malloc — all stack/static.
*/
#include "formatter.h"
#include <string.h>
/* ── helpers ────────────────────────────────────────────────────── */
static inline bool is_upper(char c) { return c >= 'A' && c <= 'Z'; }
static inline bool is_lower(char c) { return c >= 'a' && c <= 'z'; }
static inline char to_upper(char c) { return is_lower(c) ? (char)(c - 32) : c; }
static inline char to_lower(char c) { return is_upper(c) ? (char)(c + 32) : c; }
static bool str_eq(const char *a, const char *b, size_t n)
{
for (size_t i = 0; i < n; i++) {
if (a[i] != b[i]) return false;
}
return true;
}
static bool starts_with(const char *s, size_t slen, const char *prefix)
{
size_t plen = strlen(prefix);
if (slen < plen) return false;
return str_eq(s, prefix, plen);
}
/* ── output buffer append ───────────────────────────────────────── */
static void emit_char(struct steno_fmt_result *r, char c)
{
if (r->len < STENO_FMT_MAX_OUTPUT - 1) {
r->text[r->len++] = c;
}
}
/* ── apply capitalization/mode transforms to a text segment ───── */
static void emit_text_transformed(struct steno_fmt_state *state,
struct steno_fmt_result *r,
const char *text, size_t len)
{
if (len == 0) return;
for (size_t i = 0; i < len; i++) {
char c = text[i];
/* Mode transforms */
if (state->mode == STENO_MODE_CAPS) {
c = to_upper(c);
} else if (state->mode == STENO_MODE_LOWER) {
c = to_lower(c);
} else if (state->mode == STENO_MODE_TITLE) {
/* Capitalize first letter of each word */
if (i == 0 || (i > 0 && text[i - 1] == ' ')) {
c = to_upper(c);
}
}
/* One-shot transforms (first char only) */
if (i == 0) {
if (state->cap_next) {
c = to_upper(c);
state->cap_next = false;
}
if (state->upper_next) {
/* uppercase entire word — handled below */
}
if (state->lower_next) {
/* lowercase entire word — handled below */
}
}
/* upper_next: entire word */
if (state->upper_next) {
c = to_upper(c);
}
/* lower_next: entire word */
if (state->lower_next) {
c = to_lower(c);
}
emit_char(r, c);
}
state->upper_next = false;
state->lower_next = false;
}
/* ── prepend space if needed ────────────────────────────────────── */
static void maybe_space(struct steno_fmt_state *state,
struct steno_fmt_result *r)
{
if (state->space_pending && !state->suppress_space) {
emit_char(r, ' ');
}
state->suppress_space = false;
}
/* ── process a single {…} command ──────────────────────────────── */
static void process_command(struct steno_fmt_state *state,
struct steno_fmt_result *r,
const char *cmd, size_t cmd_len)
{
/* {^} — suppress space */
if (cmd_len == 1 && cmd[0] == '^') {
state->suppress_space = true;
return;
}
/* {^suffix} — attach suffix */
if (cmd_len > 1 && cmd[0] == '^') {
state->suppress_space = true;
maybe_space(state, r);
emit_text_transformed(state, r, cmd + 1, cmd_len - 1);
state->space_pending = true;
state->glue = false;
return;
}
/* {prefix^} — attach prefix */
if (cmd_len > 1 && cmd[cmd_len - 1] == '^') {
maybe_space(state, r);
emit_text_transformed(state, r, cmd, cmd_len - 1);
state->suppress_space = true;
state->space_pending = false;
state->glue = false;
return;
}
/* {-|} — capitalize next */
if (cmd_len == 2 && cmd[0] == '-' && cmd[1] == '|') {
state->cap_next = true;
return;
}
/* {~|} — carry capitalize */
if (cmd_len == 2 && cmd[0] == '~' && cmd[1] == '|') {
state->cap_next = true;
return;
}
/* {*-|} — retro capitalize (backspace + re-emit) */
if (cmd_len == 3 && cmd[0] == '*' && cmd[1] == '-' && cmd[2] == '|') {
/* Signal retro-capitalize — simplified: set flag, caller handles */
r->backspaces = 1;
return;
}
/* {*!} — retro delete space */
if (cmd_len == 2 && cmd[0] == '*' && cmd[1] == '!') {
r->backspaces = 1;
return;
}
/* {*?} — retro insert space */
if (cmd_len == 2 && cmd[0] == '*' && cmd[1] == '?') {
/* Would need undo context; simplified signal */
return;
}
/* {*} — undo */
if (cmd_len == 1 && cmd[0] == '*') {
r->is_undo = true;
r->is_command_only = true;
return;
}
/* {.} {,} {?} {!} {;} {:} — punctuation: attach to prev, cap next for sentence-enders */
if (cmd_len == 1 && (cmd[0] == '.' || cmd[0] == ',' ||
cmd[0] == '?' || cmd[0] == '!' ||
cmd[0] == ';' || cmd[0] == ':')) {
state->suppress_space = true;
maybe_space(state, r);
emit_char(r, cmd[0]);
state->space_pending = true;
state->suppress_space = false;
/* Sentence-ending punctuation capitalizes next */
if (cmd[0] == '.' || cmd[0] == '?' || cmd[0] == '!') {
state->cap_next = true;
}
state->glue = false;
return;
}
/* {#...} — key combo (emit nothing for now, could extend) */
if (cmd_len > 0 && cmd[0] == '#') {
r->is_command_only = true;
return;
}
/* {&letter} — fingerspelling */
if (cmd_len >= 2 && cmd[0] == '&') {
/* Glue: no space between glue strokes */
if (state->glue) {
state->suppress_space = true;
}
maybe_space(state, r);
emit_text_transformed(state, r, cmd + 1, cmd_len - 1);
state->space_pending = true;
state->glue = true;
state->suppress_space = false;
return;
}
/* {&} — bare glue */
if (cmd_len == 1 && cmd[0] == '&') {
state->glue = true;
state->suppress_space = true;
return;
}
/* {MODE:CAPS} {MODE:TITLE} {MODE:LOWER} {MODE:RESET} */
if (starts_with(cmd, cmd_len, "MODE:")) {
const char *mode_str = cmd + 5;
size_t mode_len = cmd_len - 5;
if (mode_len == 4 && str_eq(mode_str, "CAPS", 4)) {
state->mode = STENO_MODE_CAPS;
} else if (mode_len == 5 && str_eq(mode_str, "TITLE", 5)) {
state->mode = STENO_MODE_TITLE;
} else if (mode_len == 5 && str_eq(mode_str, "LOWER", 5)) {
state->mode = STENO_MODE_LOWER;
} else if (mode_len == 5 && str_eq(mode_str, "RESET", 5)) {
state->mode = STENO_MODE_NORMAL;
}
return;
}
/* {<} — uppercase next word */
if (cmd_len == 1 && cmd[0] == '<') {
state->upper_next = true;
return;
}
/* {>} — lowercase next word */
if (cmd_len == 1 && cmd[0] == '>') {
state->lower_next = true;
return;
}
}
/* ── main entry point ──────────────────────────────────────────── */
void steno_fmt_init(struct steno_fmt_state *state)
{
memset(state, 0, sizeof(*state));
state->space_pending = false;
state->cap_next = false;
state->upper_next = false;
state->lower_next = false;
state->suppress_space = false;
state->glue = false;
state->mode = STENO_MODE_NORMAL;
}
void steno_fmt_process(struct steno_fmt_state *state,
const char *translation,
struct steno_fmt_result *result)
{
memset(result, 0, sizeof(*result));
if (!translation || !translation[0]) {
result->is_command_only = true;
return;
}
size_t tlen = strlen(translation);
const char *p = translation;
const char *end = translation + tlen;
bool emitted_text = false;
bool had_command = false;
bool this_glue = false;
while (p < end) {
/* Escaped braces: \{ \} */
if (p[0] == '\\' && p + 1 < end && (p[1] == '{' || p[1] == '}')) {
maybe_space(state, result);
emit_text_transformed(state, result, p + 1, 1);
state->space_pending = true;
state->glue = false;
emitted_text = true;
p += 2;
continue;
}
/* Command: {…} */
if (p[0] == '{') {
const char *close = p + 1;
while (close < end && *close != '}') {
close++;
}
if (close < end) {
const char *cmd = p + 1;
size_t cmd_len = (size_t)(close - cmd);
process_command(state, result, cmd, cmd_len);
had_command = true;
if (state->glue) this_glue = true;
p = close + 1;
continue;
}
}
/* Literal text segment — find end (next '{' or '\' or end) */
const char *seg_start = p;
while (p < end && p[0] != '{' &&
!(p[0] == '\\' && p + 1 < end && (p[1] == '{' || p[1] == '}'))) {
p++;
}
size_t seg_len = (size_t)(p - seg_start);
if (seg_len > 0) {
if (!this_glue) {
maybe_space(state, result);
}
emit_text_transformed(state, result, seg_start, seg_len);
state->space_pending = true;
if (!this_glue) {
state->glue = false;
}
emitted_text = true;
}
}
result->text[result->len] = '\0';
result->is_command_only = !emitted_text && had_command && result->len == 0;
}

46
src/formatter.h Normal file
View file

@ -0,0 +1,46 @@
#ifndef STENO_FORMATTER_H
#define STENO_FORMATTER_H
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
/* Max output length from a single translation */
#define STENO_FMT_MAX_OUTPUT 128
/* Formatting mode */
enum steno_fmt_mode {
STENO_MODE_NORMAL = 0,
STENO_MODE_CAPS,
STENO_MODE_TITLE,
STENO_MODE_LOWER,
};
/* State persists between translations */
struct steno_fmt_state {
bool space_pending; /* insert space before next word */
bool cap_next; /* capitalize next word */
bool upper_next; /* uppercase next word */
bool lower_next; /* lowercase next word */
bool suppress_space; /* {^} suppress next space */
bool glue; /* in fingerspelling/glue sequence */
enum steno_fmt_mode mode;
};
/* Result of formatting one translation */
struct steno_fmt_result {
char text[STENO_FMT_MAX_OUTPUT]; /* output text to emit */
uint8_t len; /* length of text */
uint8_t backspaces; /* backspaces to send BEFORE text */
bool is_command_only; /* true if no text output (pure command) */
bool is_undo; /* true if this is an undo command */
};
void steno_fmt_init(struct steno_fmt_state *state);
/* Format a raw translation string. Updates state, fills result. */
void steno_fmt_process(struct steno_fmt_state *state,
const char *translation,
struct steno_fmt_result *result);
#endif

167
src/output.c Normal file
View file

@ -0,0 +1,167 @@
#include "output.h"
#include <zephyr/kernel.h>
#include <zephyr/logging/log.h>
#include <zmk/hid.h>
#include <zmk/endpoints.h>
#include <zmk/events/keycode_state_changed.h>
LOG_MODULE_DECLARE(zmk, CONFIG_ZMK_LOG_LEVEL);
struct hid_map {
uint8_t keycode;
bool shift;
};
static const struct hid_map ASCII_TO_HID[128] = {
[' '] = {0x2C, false}, /* space */
['!'] = {0x1E, true}, /* shift+1 */
['"'] = {0x34, true}, /* shift+' */
['#'] = {0x20, true},
['$'] = {0x21, true},
['%'] = {0x22, true},
['&'] = {0x24, true},
['\''] = {0x34, false},
['('] = {0x26, true},
[')'] = {0x27, true},
['*'] = {0x25, true},
['+'] = {0x2E, true},
[','] = {0x36, false},
['-'] = {0x2D, false},
['.'] = {0x37, false},
['/'] = {0x38, false},
['0'] = {0x27, false},
['1'] = {0x1E, false},
['2'] = {0x1F, false},
['3'] = {0x20, false},
['4'] = {0x21, false},
['5'] = {0x22, false},
['6'] = {0x23, false},
['7'] = {0x24, false},
['8'] = {0x25, false},
['9'] = {0x26, false},
[':'] = {0x33, true},
[';'] = {0x33, false},
['<'] = {0x36, true},
['='] = {0x2E, false},
['>'] = {0x37, true},
['?'] = {0x38, true},
['@'] = {0x1F, true},
['A'] = {0x04, true},
['B'] = {0x05, true},
['C'] = {0x06, true},
['D'] = {0x07, true},
['E'] = {0x08, true},
['F'] = {0x09, true},
['G'] = {0x0A, true},
['H'] = {0x0B, true},
['I'] = {0x0C, true},
['J'] = {0x0D, true},
['K'] = {0x0E, true},
['L'] = {0x0F, true},
['M'] = {0x10, true},
['N'] = {0x11, true},
['O'] = {0x12, true},
['P'] = {0x13, true},
['Q'] = {0x14, true},
['R'] = {0x15, true},
['S'] = {0x16, true},
['T'] = {0x17, true},
['U'] = {0x18, true},
['V'] = {0x19, true},
['W'] = {0x1A, true},
['X'] = {0x1B, true},
['Y'] = {0x1C, true},
['Z'] = {0x1D, true},
['['] = {0x2F, false},
['\\'] = {0x31, false},
[']'] = {0x30, false},
['^'] = {0x23, true},
['_'] = {0x2D, true},
['`'] = {0x35, false},
['a'] = {0x04, false},
['b'] = {0x05, false},
['c'] = {0x06, false},
['d'] = {0x07, false},
['e'] = {0x08, false},
['f'] = {0x09, false},
['g'] = {0x0A, false},
['h'] = {0x0B, false},
['i'] = {0x0C, false},
['j'] = {0x0D, false},
['k'] = {0x0E, false},
['l'] = {0x0F, false},
['m'] = {0x10, false},
['n'] = {0x11, false},
['o'] = {0x12, false},
['p'] = {0x13, false},
['q'] = {0x14, false},
['r'] = {0x15, false},
['s'] = {0x16, false},
['t'] = {0x17, false},
['u'] = {0x18, false},
['v'] = {0x19, false},
['w'] = {0x1A, false},
['x'] = {0x1B, false},
['y'] = {0x1C, false},
['z'] = {0x1D, false},
['{'] = {0x2F, true},
['|'] = {0x31, true},
['}'] = {0x30, true},
['~'] = {0x35, true},
};
#define HID_BACKSPACE 0x2A
#define HID_RETURN 0x28
#define HID_LSHIFT 0xE1
static void tap_key(uint8_t keycode, bool shift)
{
if (shift) {
raise_zmk_keycode_state_changed((struct zmk_keycode_state_changed){
.usage_page = 0x07, .keycode = HID_LSHIFT,
.implicit_modifiers = 0, .explicit_modifiers = 0,
.state = true, .timestamp = k_uptime_get()});
}
raise_zmk_keycode_state_changed((struct zmk_keycode_state_changed){
.usage_page = 0x07, .keycode = keycode,
.implicit_modifiers = 0, .explicit_modifiers = 0,
.state = true, .timestamp = k_uptime_get()});
raise_zmk_keycode_state_changed((struct zmk_keycode_state_changed){
.usage_page = 0x07, .keycode = keycode,
.implicit_modifiers = 0, .explicit_modifiers = 0,
.state = false, .timestamp = k_uptime_get()});
if (shift) {
raise_zmk_keycode_state_changed((struct zmk_keycode_state_changed){
.usage_page = 0x07, .keycode = HID_LSHIFT,
.implicit_modifiers = 0, .explicit_modifiers = 0,
.state = false, .timestamp = k_uptime_get()});
}
}
void steno_output_send(const char *text, size_t len)
{
for (size_t i = 0; i < len; i++) {
unsigned char c = (unsigned char)text[i];
if (c == '\n') {
tap_key(HID_RETURN, false);
continue;
}
if (c >= 128 || ASCII_TO_HID[c].keycode == 0) {
LOG_WRN("Skipping non-ASCII char 0x%02X", c);
continue;
}
tap_key(ASCII_TO_HID[c].keycode, ASCII_TO_HID[c].shift);
}
}
void steno_output_backspace(int count)
{
for (int i = 0; i < count; i++) {
tap_key(HID_BACKSPACE, false);
}
}

11
src/output.h Normal file
View file

@ -0,0 +1,11 @@
#ifndef STENO_OUTPUT_H
#define STENO_OUTPUT_H
#include <stddef.h>
#include <stdint.h>
void steno_output_send(const char *text, size_t len);
void steno_output_backspace(int count);
#endif

167
src/split_cache.c Normal file
View file

@ -0,0 +1,167 @@
/*
* Copyright (c) 2024 zmk-steno-engine contributors
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*
* Licensed under the PolyForm Noncommercial License 1.0.0;
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* https://polyformproject.org/licenses/noncommercial/1.0.0
*/
#include <string.h>
#include "split_cache.h"
/* FNV-1a hash over stroke bytes */
static uint32_t hash_strokes(const uint32_t *strokes, uint8_t count)
{
uint32_t hash = 2166136261u; /* FNV offset basis */
for (uint8_t i = 0; i < count; i++) {
uint32_t s = strokes[i];
for (int b = 0; b < 4; b++) {
hash ^= (s & 0xFF);
hash *= 16777619u; /* FNV prime */
s >>= 8;
}
}
return hash;
}
static bool strokes_match(const struct cache_entry *entry,
const uint32_t *strokes, uint8_t count)
{
if (entry->stroke_count != count) {
return false;
}
return memcmp(entry->strokes, strokes, count * sizeof(uint32_t)) == 0;
}
void split_cache_init(struct split_cache *cache)
{
memset(cache->entries, 0,
sizeof(struct cache_entry) * CONFIG_STENO_SPLIT_CACHE_SIZE);
cache->access_counter = 0;
cache->hits = 0;
cache->misses = 0;
}
bool split_cache_lookup(struct split_cache *cache, const uint32_t *strokes,
uint8_t count, char *result, size_t result_size,
bool *has_prefix)
{
uint32_t h = hash_strokes(strokes, count);
for (int i = 0; i < CONFIG_STENO_SPLIT_CACHE_SIZE; i++) {
struct cache_entry *e = &cache->entries[i];
if (!e->valid) {
continue;
}
if (e->key_hash == h && strokes_match(e, strokes, count)) {
/* Hit */
cache->access_counter++;
e->access_count = cache->access_counter;
cache->hits++;
if (has_prefix) {
*has_prefix = e->has_prefix;
}
if (result && result_size > 0) {
size_t len = strlen(e->translation);
if (len >= result_size) {
len = result_size - 1;
}
memcpy(result, e->translation, len);
result[len] = '\0';
}
return true;
}
}
cache->misses++;
return false;
}
void split_cache_insert(struct split_cache *cache, const uint32_t *strokes,
uint8_t count, const char *translation, bool has_prefix)
{
if (count == 0 || count > 8) {
return;
}
uint32_t h = hash_strokes(strokes, count);
/* Check if already present → update */
for (int i = 0; i < CONFIG_STENO_SPLIT_CACHE_SIZE; i++) {
struct cache_entry *e = &cache->entries[i];
if (e->valid && e->key_hash == h && strokes_match(e, strokes, count)) {
/* Update existing entry */
if (translation) {
size_t len = strlen(translation);
if (len >= SPLIT_CACHE_VALUE_SIZE) {
len = SPLIT_CACHE_VALUE_SIZE - 1;
}
memcpy(e->translation, translation, len);
e->translation[len] = '\0';
}
e->has_prefix = has_prefix;
cache->access_counter++;
e->access_count = cache->access_counter;
return;
}
}
/* Find empty slot or LRU victim */
int target = -1;
uint32_t min_access = UINT32_MAX;
for (int i = 0; i < CONFIG_STENO_SPLIT_CACHE_SIZE; i++) {
if (!cache->entries[i].valid) {
target = i;
break;
}
if (cache->entries[i].access_count < min_access) {
min_access = cache->entries[i].access_count;
target = i;
}
}
if (target < 0) {
target = 0; /* fallback: should never happen if cache size > 0 */
}
struct cache_entry *e = &cache->entries[target];
e->key_hash = h;
e->stroke_count = count;
memcpy(e->strokes, strokes, count * sizeof(uint32_t));
if (translation) {
size_t len = strlen(translation);
if (len >= SPLIT_CACHE_VALUE_SIZE) {
len = SPLIT_CACHE_VALUE_SIZE - 1;
}
memcpy(e->translation, translation, len);
e->translation[len] = '\0';
} else {
e->translation[0] = '\0';
}
e->has_prefix = has_prefix;
e->valid = true;
cache->access_counter++;
e->access_count = cache->access_counter;
}
void split_cache_invalidate(struct split_cache *cache)
{
for (int i = 0; i < CONFIG_STENO_SPLIT_CACHE_SIZE; i++) {
cache->entries[i].valid = false;
}
cache->access_counter = 0;
cache->hits = 0;
cache->misses = 0;
}

46
src/split_cache.h Normal file
View file

@ -0,0 +1,46 @@
/*
* Copyright (c) 2024 zmk-steno-engine contributors
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*
* Licensed under the PolyForm Noncommercial License 1.0.0;
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* https://polyformproject.org/licenses/noncommercial/1.0.0
*/
#ifndef SPLIT_CACHE_H
#define SPLIT_CACHE_H
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#define SPLIT_CACHE_KEY_SIZE 24 /* max 8 strokes * 3 bytes */
#define SPLIT_CACHE_VALUE_SIZE 128 /* max translation length */
struct cache_entry {
uint32_t key_hash;
uint8_t stroke_count;
uint32_t strokes[8];
char translation[SPLIT_CACHE_VALUE_SIZE];
bool has_prefix;
bool valid;
uint32_t access_count;
};
struct split_cache {
struct cache_entry entries[CONFIG_STENO_SPLIT_CACHE_SIZE];
uint32_t access_counter;
uint32_t hits;
uint32_t misses;
};
void split_cache_init(struct split_cache *cache);
bool split_cache_lookup(struct split_cache *cache, const uint32_t *strokes,
uint8_t count, char *result, size_t result_size,
bool *has_prefix);
void split_cache_insert(struct split_cache *cache, const uint32_t *strokes,
uint8_t count, const char *translation, bool has_prefix);
void split_cache_invalidate(struct split_cache *cache);
#endif /* SPLIT_CACHE_H */

465
src/split_dict.c Normal file
View file

@ -0,0 +1,465 @@
/*
* Copyright (c) 2024 zmk-steno-engine contributors
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*
* Licensed under the PolyForm Noncommercial License 1.0.0;
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* https://polyformproject.org/licenses/noncommercial/1.0.0
*/
#include <string.h>
#include <zephyr/kernel.h>
#include <zephyr/bluetooth/bluetooth.h>
#include <zephyr/bluetooth/gatt.h>
#include <zephyr/bluetooth/conn.h>
#include <zephyr/bluetooth/uuid.h>
#include <zephyr/logging/log.h>
#include "split_dict.h"
#include "split_cache.h"
LOG_MODULE_REGISTER(split_dict, CONFIG_STENO_SPLIT_LOG_LEVEL);
/* Semaphore for blocking on BLE response */
static K_SEM_DEFINE(response_sem, 0, 1);
/* Current pending response state */
static uint8_t pending_seq;
static uint8_t response_buf[256];
static uint16_t response_len;
static uint8_t seq_counter;
/* Cache instance */
static struct split_cache dict_cache;
/* External trie lookup (peripheral side) */
extern int trie_lookup(const uint32_t *strokes, uint8_t count,
char *result, size_t result_size);
extern bool trie_has_prefix(const uint32_t *strokes, uint8_t count);
/* --- Helpers --- */
static void encode_strokes(const uint32_t *strokes, uint8_t count, uint8_t *out)
{
for (uint8_t i = 0; i < count; i++) {
out[i * 3 + 0] = (strokes[i] >> 16) & 0xFF;
out[i * 3 + 1] = (strokes[i] >> 8) & 0xFF;
out[i * 3 + 2] = strokes[i] & 0xFF;
}
}
static void decode_strokes(const uint8_t *in, uint8_t count, uint32_t *strokes)
{
for (uint8_t i = 0; i < count; i++) {
strokes[i] = ((uint32_t)in[i * 3 + 0] << 16) |
((uint32_t)in[i * 3 + 1] << 8) |
(uint32_t)in[i * 3 + 2];
}
}
/* --- GATT Write Callbacks (peripheral side handlers) --- */
static ssize_t dict_query_write_cb(struct bt_conn *conn,
const struct bt_gatt_attr *attr,
const void *buf, uint16_t len,
uint16_t offset, uint8_t flags)
{
const struct steno_query_pkt *pkt = buf;
if (len < sizeof(struct steno_query_pkt)) {
LOG_WRN("Query pkt too short: %u", len);
return BT_GATT_ERR(BT_ATT_ERR_INVALID_ATTRIBUTE_LEN);
}
uint8_t stroke_count = pkt->stroke_count;
uint16_t expected = sizeof(struct steno_query_pkt) + stroke_count * 3;
if (len < expected) {
LOG_WRN("Query pkt truncated: got %u, need %u", len, expected);
return BT_GATT_ERR(BT_ATT_ERR_INVALID_ATTRIBUTE_LEN);
}
uint32_t strokes[8];
if (stroke_count > 8) {
stroke_count = 8;
}
decode_strokes(pkt->strokes, stroke_count, strokes);
/* Build response */
struct steno_response_pkt *resp = (struct steno_response_pkt *)response_buf;
resp->msg_type = STENO_MSG_RESPONSE;
resp->seq = pkt->seq;
char translation[128];
int ret = trie_lookup(strokes, stroke_count, translation, sizeof(translation));
if (ret > 0) {
resp->status = STENO_STATUS_FOUND;
resp->data_len = (uint16_t)ret;
memcpy(resp->data, translation, ret);
response_len = sizeof(struct steno_response_pkt) + ret;
} else {
resp->status = STENO_STATUS_NOT_FOUND;
resp->data_len = 0;
response_len = sizeof(struct steno_response_pkt);
}
/* Notify central with response */
bt_gatt_notify(conn, attr, response_buf, response_len);
return len;
}
static ssize_t dict_prefix_write_cb(struct bt_conn *conn,
const struct bt_gatt_attr *attr,
const void *buf, uint16_t len,
uint16_t offset, uint8_t flags)
{
const struct steno_query_pkt *pkt = buf;
if (len < sizeof(struct steno_query_pkt)) {
return BT_GATT_ERR(BT_ATT_ERR_INVALID_ATTRIBUTE_LEN);
}
uint8_t stroke_count = pkt->stroke_count;
if (stroke_count > 8) {
stroke_count = 8;
}
uint32_t strokes[8];
decode_strokes(pkt->strokes, stroke_count, strokes);
struct steno_response_pkt *resp = (struct steno_response_pkt *)response_buf;
resp->msg_type = STENO_MSG_RESPONSE;
resp->seq = pkt->seq;
resp->data_len = 0;
if (trie_has_prefix(strokes, stroke_count)) {
resp->status = STENO_STATUS_PREFIX_ONLY;
} else {
resp->status = STENO_STATUS_NOT_FOUND;
}
response_len = sizeof(struct steno_response_pkt);
bt_gatt_notify(conn, attr, response_buf, response_len);
return len;
}
static ssize_t dict_batch_write_cb(struct bt_conn *conn,
const struct bt_gatt_attr *attr,
const void *buf, uint16_t len,
uint16_t offset, uint8_t flags)
{
const struct steno_batch_query_pkt *pkt = buf;
if (len < sizeof(struct steno_batch_query_pkt)) {
return BT_GATT_ERR(BT_ATT_ERR_INVALID_ATTRIBUTE_LEN);
}
LOG_DBG("Batch query: %u queries", pkt->query_count);
/* Process each sub-query packed in queries[] */
uint16_t pos = 0;
const uint8_t *data = pkt->queries;
uint16_t data_len = len - sizeof(struct steno_batch_query_pkt);
for (uint8_t q = 0; q < pkt->query_count && pos < data_len; q++) {
if (pos >= data_len) {
break;
}
uint8_t stroke_count = data[pos];
pos++;
if (stroke_count > 8) {
stroke_count = 8;
}
if (pos + stroke_count * 3 > data_len) {
break;
}
uint32_t strokes[8];
decode_strokes(&data[pos], stroke_count, strokes);
pos += stroke_count * 3;
/* Lookup and send individual response per query */
struct steno_response_pkt *resp = (struct steno_response_pkt *)response_buf;
resp->msg_type = STENO_MSG_RESPONSE;
resp->seq = pkt->seq;
char translation[128];
int ret = trie_lookup(strokes, stroke_count, translation, sizeof(translation));
if (ret > 0) {
resp->status = STENO_STATUS_FOUND;
resp->data_len = (uint16_t)ret;
memcpy(resp->data, translation, ret);
response_len = sizeof(struct steno_response_pkt) + ret;
} else {
resp->status = STENO_STATUS_NOT_FOUND;
resp->data_len = 0;
response_len = sizeof(struct steno_response_pkt);
}
bt_gatt_notify(conn, attr, response_buf, response_len);
}
return len;
}
/* --- Notification callback (central side) --- */
static uint8_t notify_cb(struct bt_conn *conn,
struct bt_gatt_subscribe_params *params,
const void *data, uint16_t length)
{
if (!data) {
LOG_DBG("Notification unsubscribed");
return BT_GATT_ITER_STOP;
}
const struct steno_response_pkt *resp = data;
if (length < sizeof(struct steno_response_pkt)) {
LOG_WRN("Response too short");
return BT_GATT_ITER_CONTINUE;
}
if (resp->seq == pending_seq) {
memcpy(response_buf, data, length);
response_len = length;
k_sem_give(&response_sem);
}
return BT_GATT_ITER_CONTINUE;
}
/* --- GATT Service Definition --- */
BT_GATT_SERVICE_DEFINE(steno_dict_svc,
BT_GATT_PRIMARY_SERVICE(STENO_UUID_SERVICE),
/* Dict Query characteristic: write + notify */
BT_GATT_CHARACTERISTIC(STENO_UUID_DICT_QUERY,
BT_GATT_CHRC_WRITE | BT_GATT_CHRC_NOTIFY,
BT_GATT_PERM_WRITE,
NULL, dict_query_write_cb, NULL),
BT_GATT_CCC(NULL, BT_GATT_PERM_READ | BT_GATT_PERM_WRITE),
/* Dict Prefix characteristic: write + notify */
BT_GATT_CHARACTERISTIC(STENO_UUID_DICT_PREFIX,
BT_GATT_CHRC_WRITE | BT_GATT_CHRC_NOTIFY,
BT_GATT_PERM_WRITE,
NULL, dict_prefix_write_cb, NULL),
BT_GATT_CCC(NULL, BT_GATT_PERM_READ | BT_GATT_PERM_WRITE),
/* Dict Batch characteristic: write + notify */
BT_GATT_CHARACTERISTIC(STENO_UUID_DICT_BATCH,
BT_GATT_CHRC_WRITE | BT_GATT_CHRC_NOTIFY,
BT_GATT_PERM_WRITE,
NULL, dict_batch_write_cb, NULL),
BT_GATT_CCC(NULL, BT_GATT_PERM_READ | BT_GATT_PERM_WRITE),
);
/* --- Central-side API --- */
/* Connection handle for GATT writes (set externally or via connection cb) */
static struct bt_conn *split_conn;
static struct bt_gatt_subscribe_params subscribe_params;
int split_dict_lookup(const uint32_t *strokes, uint8_t count,
char *result, size_t result_size)
{
if (count == 0 || count > 8) {
return -EINVAL;
}
/* Check cache first */
bool has_prefix;
if (split_cache_lookup(&dict_cache, strokes, count, result, result_size, &has_prefix)) {
LOG_DBG("Cache hit for %u strokes", count);
return strlen(result);
}
if (!split_conn) {
LOG_ERR("No split connection");
return -ENOTCONN;
}
/* Build query packet */
uint8_t pkt_buf[sizeof(struct steno_query_pkt) + 8 * 3];
struct steno_query_pkt *pkt = (struct steno_query_pkt *)pkt_buf;
pkt->msg_type = STENO_MSG_QUERY;
pkt->seq = seq_counter++;
pkt->stroke_count = count;
encode_strokes(strokes, count, pkt->strokes);
pending_seq = pkt->seq;
k_sem_reset(&response_sem);
uint16_t pkt_len = sizeof(struct steno_query_pkt) + count * 3;
/* Send via GATT write */
int err = bt_gatt_write_without_response(split_conn, 0, pkt_buf, pkt_len, false);
if (err) {
LOG_ERR("GATT write failed: %d", err);
return err;
}
/* Wait for response */
err = k_sem_take(&response_sem, K_MSEC(CONFIG_STENO_SPLIT_TIMEOUT_MS));
if (err) {
LOG_WRN("Response timeout");
return -ETIMEDOUT;
}
/* Decode response */
const struct steno_response_pkt *resp = (const struct steno_response_pkt *)response_buf;
if (resp->status == STENO_STATUS_FOUND) {
uint16_t copy_len = resp->data_len;
if (copy_len >= result_size) {
copy_len = result_size - 1;
}
memcpy(result, resp->data, copy_len);
result[copy_len] = '\0';
/* Cache the result */
split_cache_insert(&dict_cache, strokes, count, result, false);
return copy_len;
}
return -ENOENT;
}
bool split_dict_has_prefix(const uint32_t *strokes, uint8_t count)
{
if (count == 0 || count > 8) {
return false;
}
/* Check cache */
bool has_prefix;
char dummy[1];
if (split_cache_lookup(&dict_cache, strokes, count, dummy, sizeof(dummy), &has_prefix)) {
return has_prefix;
}
if (!split_conn) {
return false;
}
uint8_t pkt_buf[sizeof(struct steno_query_pkt) + 8 * 3];
struct steno_query_pkt *pkt = (struct steno_query_pkt *)pkt_buf;
pkt->msg_type = STENO_MSG_PREFIX;
pkt->seq = seq_counter++;
pkt->stroke_count = count;
encode_strokes(strokes, count, pkt->strokes);
pending_seq = pkt->seq;
k_sem_reset(&response_sem);
uint16_t pkt_len = sizeof(struct steno_query_pkt) + count * 3;
int err = bt_gatt_write_without_response(split_conn, 0, pkt_buf, pkt_len, false);
if (err) {
return false;
}
err = k_sem_take(&response_sem, K_MSEC(CONFIG_STENO_SPLIT_TIMEOUT_MS));
if (err) {
return false;
}
const struct steno_response_pkt *resp = (const struct steno_response_pkt *)response_buf;
return resp->status == STENO_STATUS_PREFIX_ONLY;
}
int split_dict_batch_lookup(const uint32_t **stroke_seqs, const uint8_t *counts,
uint8_t num_queries, struct steno_batch_result *results)
{
if (num_queries == 0 || !split_conn) {
return -EINVAL;
}
/* Build batch packet */
uint8_t pkt_buf[256];
struct steno_batch_query_pkt *pkt = (struct steno_batch_query_pkt *)pkt_buf;
pkt->msg_type = STENO_MSG_BATCH;
pkt->seq = seq_counter++;
pkt->query_count = num_queries;
uint16_t pos = 0;
for (uint8_t q = 0; q < num_queries; q++) {
uint8_t cnt = counts[q];
if (cnt > 8) {
cnt = 8;
}
pkt->queries[pos] = cnt;
pos++;
encode_strokes(stroke_seqs[q], cnt, &pkt->queries[pos]);
pos += cnt * 3;
}
uint16_t pkt_len = sizeof(struct steno_batch_query_pkt) + pos;
pending_seq = pkt->seq;
k_sem_reset(&response_sem);
int err = bt_gatt_write_without_response(split_conn, 0, pkt_buf, pkt_len, false);
if (err) {
return err;
}
/* Collect responses for each query */
for (uint8_t q = 0; q < num_queries; q++) {
err = k_sem_take(&response_sem, K_MSEC(CONFIG_STENO_SPLIT_TIMEOUT_MS));
if (err) {
results[q].status = STENO_STATUS_ERROR;
continue;
}
const struct steno_response_pkt *resp =
(const struct steno_response_pkt *)response_buf;
results[q].status = resp->status;
results[q].has_prefix = (resp->status == STENO_STATUS_PREFIX_ONLY);
if (resp->status == STENO_STATUS_FOUND && resp->data_len > 0) {
uint16_t copy_len = resp->data_len;
if (copy_len >= sizeof(results[q].translation)) {
copy_len = sizeof(results[q].translation) - 1;
}
memcpy(results[q].translation, resp->data, copy_len);
results[q].translation[copy_len] = '\0';
results[q].translation_len = copy_len;
} else {
results[q].translation[0] = '\0';
results[q].translation_len = 0;
}
}
return 0;
}
int split_dict_init(void)
{
split_cache_init(&dict_cache);
seq_counter = 0;
split_conn = NULL;
LOG_INF("Split dict initialized");
return 0;
}
int split_dict_gatt_register(void)
{
/* GATT service registered statically via BT_GATT_SERVICE_DEFINE */
LOG_INF("Split dict GATT service registered");
return 0;
}

99
src/split_dict.h Normal file
View file

@ -0,0 +1,99 @@
/*
* Copyright (c) 2024 zmk-steno-engine contributors
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*
* Licensed under the PolyForm Noncommercial License 1.0.0;
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* https://polyformproject.org/licenses/noncommercial/1.0.0
*/
#ifndef SPLIT_DICT_H
#define SPLIT_DICT_H
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#include <zephyr/bluetooth/uuid.h>
/* Custom 128-bit UUID base for steno GATT service
* Base: 7374656e-6f00-4000-8000-000000000000 */
#define STENO_UUID_BASE \
BT_UUID_DECLARE_128(BT_UUID_128_ENCODE( \
0x7374656e, 0x6f00, 0x4000, 0x8000, 0x000000000000))
#define STENO_UUID_SERVICE \
BT_UUID_DECLARE_128(BT_UUID_128_ENCODE( \
0x7374656e, 0x6f00, 0x4000, 0x8000, 0x000000000001))
#define STENO_UUID_DICT_QUERY \
BT_UUID_DECLARE_128(BT_UUID_128_ENCODE( \
0x7374656e, 0x6f00, 0x4000, 0x8000, 0x000000000002))
#define STENO_UUID_DICT_PREFIX \
BT_UUID_DECLARE_128(BT_UUID_128_ENCODE( \
0x7374656e, 0x6f00, 0x4000, 0x8000, 0x000000000003))
#define STENO_UUID_DICT_BATCH \
BT_UUID_DECLARE_128(BT_UUID_128_ENCODE( \
0x7374656e, 0x6f00, 0x4000, 0x8000, 0x000000000004))
/* Message types */
enum steno_msg_type {
STENO_MSG_QUERY = 0x01,
STENO_MSG_PREFIX = 0x02,
STENO_MSG_BATCH = 0x03,
STENO_MSG_RESPONSE = 0x80,
};
/* Status codes */
enum steno_status {
STENO_STATUS_FOUND = 0,
STENO_STATUS_NOT_FOUND = 1,
STENO_STATUS_PREFIX_ONLY = 2,
STENO_STATUS_ERROR = 3,
};
/* Packet structures */
struct steno_query_pkt {
uint8_t msg_type;
uint8_t seq;
uint8_t stroke_count;
uint8_t strokes[]; /* 3 bytes per stroke (24-bit packed) */
} __packed;
struct steno_response_pkt {
uint8_t msg_type;
uint8_t seq;
uint8_t status;
uint16_t data_len;
uint8_t data[]; /* translation string (UTF-8, not null-terminated) */
} __packed;
struct steno_batch_query_pkt {
uint8_t msg_type;
uint8_t seq;
uint8_t query_count;
uint8_t queries[]; /* packed steno_query_pkt entries (without msg_type/seq) */
} __packed;
/* Batch result entry */
struct steno_batch_result {
uint8_t status;
char translation[128];
uint16_t translation_len;
bool has_prefix;
};
/* API */
int split_dict_init(void);
int split_dict_lookup(const uint32_t *strokes, uint8_t count,
char *result, size_t result_size);
bool split_dict_has_prefix(const uint32_t *strokes, uint8_t count);
int split_dict_batch_lookup(const uint32_t **stroke_seqs, const uint8_t *counts,
uint8_t num_queries, struct steno_batch_result *results);
/* GATT service registration */
int split_dict_gatt_register(void);
#endif /* SPLIT_DICT_H */

148
src/trie.c Normal file
View file

@ -0,0 +1,148 @@
#include "trie.h"
#include <string.h>
static const uint8_t *dict_data;
static const struct steno_dict_header *dict_hdr;
static const uint8_t *entry_array;
static const char *string_table;
static size_t entry_stride;
int steno_trie_init(const uint8_t *data, size_t len)
{
if (!data || len < sizeof(struct steno_dict_header)) {
return -1;
}
dict_hdr = (const struct steno_dict_header *)data;
if (dict_hdr->magic != STENO_DICT_MAGIC) {
return -2;
}
if (dict_hdr->version != 1) {
return -3;
}
if (dict_hdr->max_strokes == 0 || dict_hdr->max_strokes > 16) {
return -4;
}
entry_stride = (size_t)dict_hdr->max_strokes * 4 + 4;
size_t entries_end = sizeof(struct steno_dict_header) +
(size_t)dict_hdr->entry_count * entry_stride;
if (entries_end > len || dict_hdr->strings_offset > len) {
return -5;
}
dict_data = data;
entry_array = data + sizeof(struct steno_dict_header);
string_table = (const char *)(data + dict_hdr->strings_offset);
return 0;
}
static const uint8_t *get_entry(uint32_t idx)
{
return entry_array + (size_t)idx * entry_stride;
}
static int cmp_strokes(const uint32_t *a, uint8_t a_count,
const uint8_t *entry_bytes, uint8_t max_s)
{
const uint32_t *b = (const uint32_t *)entry_bytes;
uint8_t b_count = 0;
for (uint8_t i = 0; i < max_s; i++) {
if (b[i] != 0) {
b_count = i + 1;
}
}
uint8_t min_count = a_count < b_count ? a_count : b_count;
for (uint8_t i = 0; i < min_count; i++) {
if (a[i] < b[i]) return -1;
if (a[i] > b[i]) return 1;
}
if (a_count < b_count) return -1;
if (a_count > b_count) return 1;
return 0;
}
const char *steno_trie_lookup(const uint32_t *strokes, uint8_t count)
{
if (!dict_hdr || !strokes || count == 0 || count > dict_hdr->max_strokes) {
return NULL;
}
uint32_t lo = 0;
uint32_t hi = dict_hdr->entry_count;
while (lo < hi) {
uint32_t mid = lo + (hi - lo) / 2;
const uint8_t *entry = get_entry(mid);
int c = cmp_strokes(strokes, count, entry, dict_hdr->max_strokes);
if (c == 0) {
uint32_t str_off;
memcpy(&str_off, entry + (size_t)dict_hdr->max_strokes * 4, 4);
return string_table + str_off;
}
if (c < 0) {
hi = mid;
} else {
lo = mid + 1;
}
}
return NULL;
}
bool steno_trie_has_prefix(const uint32_t *strokes, uint8_t count)
{
if (!dict_hdr || !strokes || count == 0 || count >= dict_hdr->max_strokes) {
return false;
}
uint32_t lo = 0;
uint32_t hi = dict_hdr->entry_count;
uint32_t first_ge = hi;
while (lo < hi) {
uint32_t mid = lo + (hi - lo) / 2;
const uint8_t *entry = get_entry(mid);
const uint32_t *e_strokes = (const uint32_t *)entry;
int cmp = 0;
for (uint8_t i = 0; i < count; i++) {
if (e_strokes[i] < strokes[i]) { cmp = -1; break; }
if (e_strokes[i] > strokes[i]) { cmp = 1; break; }
}
if (cmp >= 0) {
first_ge = mid;
hi = mid;
} else {
lo = mid + 1;
}
}
for (uint32_t idx = first_ge; idx < dict_hdr->entry_count; idx++) {
const uint8_t *entry = get_entry(idx);
const uint32_t *e_strokes = (const uint32_t *)entry;
bool prefix_match = true;
for (uint8_t i = 0; i < count; i++) {
if (e_strokes[i] != strokes[i]) {
prefix_match = false;
break;
}
}
if (!prefix_match) {
return false;
}
if (e_strokes[count] != 0) {
return true;
}
}
return false;
}

25
src/trie.h Normal file
View file

@ -0,0 +1,25 @@
#ifndef STENO_TRIE_H
#define STENO_TRIE_H
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#define STENO_DICT_MAGIC 0x4F4E5453 /* "STNO" */
struct steno_dict_header {
uint32_t magic;
uint16_t version;
uint8_t max_strokes;
uint8_t _pad;
uint32_t entry_count;
uint32_t strings_offset;
} __attribute__((packed));
int steno_trie_init(const uint8_t *data, size_t len);
const char *steno_trie_lookup(const uint32_t *strokes, uint8_t count);
bool steno_trie_has_prefix(const uint32_t *strokes, uint8_t count);
#endif

64
src/undo.c Normal file
View file

@ -0,0 +1,64 @@
/*
* Copyright (c) 2024 Afiq Zudin Hadi
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*/
#include "undo.h"
#include <string.h>
void steno_undo_init(struct stroke_history *hist)
{
memset(hist, 0, sizeof(*hist));
}
void steno_undo_push(struct stroke_history *hist,
const uint32_t *strokes, uint8_t stroke_count,
uint8_t output_len, uint8_t space_before,
uint8_t fmt_flags)
{
struct stroke_history_entry *e = &hist->entries[hist->head];
uint8_t n = stroke_count;
if (n > STENO_MAX_MULTI_STROKE) {
n = STENO_MAX_MULTI_STROKE;
}
memcpy(e->strokes, strokes, n * sizeof(uint32_t));
e->stroke_count = n;
e->output_len = output_len;
e->space_before = space_before;
e->fmt_flags = fmt_flags;
hist->head = (hist->head + 1) % CONFIG_STENO_HISTORY_SIZE;
if (hist->count < CONFIG_STENO_HISTORY_SIZE) {
hist->count++;
}
}
struct stroke_history_entry *steno_undo_pop(struct stroke_history *hist)
{
if (hist->count == 0) {
return NULL;
}
hist->head = (hist->head + CONFIG_STENO_HISTORY_SIZE - 1) % CONFIG_STENO_HISTORY_SIZE;
hist->count--;
return &hist->entries[hist->head];
}
const struct stroke_history_entry *steno_undo_peek(const struct stroke_history *hist)
{
if (hist->count == 0) {
return NULL;
}
uint16_t idx = (hist->head + CONFIG_STENO_HISTORY_SIZE - 1) % CONFIG_STENO_HISTORY_SIZE;
return &hist->entries[idx];
}
uint16_t steno_undo_count(const struct stroke_history *hist)
{
return hist->count;
}

44
src/undo.h Normal file
View file

@ -0,0 +1,44 @@
#ifndef STENO_UNDO_H
#define STENO_UNDO_H
#include <stdbool.h>
#include <stdint.h>
#ifndef CONFIG_STENO_HISTORY_SIZE
#define CONFIG_STENO_HISTORY_SIZE 100
#endif
#define STENO_MAX_MULTI_STROKE 8
struct stroke_history_entry {
uint32_t strokes[STENO_MAX_MULTI_STROKE];
uint8_t stroke_count;
uint8_t output_len; /* chars emitted (for backspace count) */
uint8_t space_before; /* 1 if space was prepended */
uint8_t fmt_flags; /* formatter state snapshot for restore */
};
struct stroke_history {
struct stroke_history_entry entries[CONFIG_STENO_HISTORY_SIZE];
uint16_t head; /* next write position */
uint16_t count; /* entries in buffer */
};
void steno_undo_init(struct stroke_history *hist);
/* Push a new entry after successful output */
void steno_undo_push(struct stroke_history *hist,
const uint32_t *strokes, uint8_t stroke_count,
uint8_t output_len, uint8_t space_before,
uint8_t fmt_flags);
/* Pop most recent entry for undo. Returns NULL if empty. */
struct stroke_history_entry *steno_undo_pop(struct stroke_history *hist);
/* Peek at most recent entry without removing. Returns NULL if empty. */
const struct stroke_history_entry *steno_undo_peek(const struct stroke_history *hist);
/* Get current count */
uint16_t steno_undo_count(const struct stroke_history *hist);
#endif

64
tests/run_tests.sh Executable file
View file

@ -0,0 +1,64 @@
#!/bin/bash
# Run MPHF dictionary tests: compile test dict, build C test, execute.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
BUILD_DIR="$SCRIPT_DIR/build"
mkdir -p "$BUILD_DIR"
echo "=== Step 1: Create test dictionary JSON ==="
cat > "$BUILD_DIR/test_dict.json" << 'DICTEOF'
{
"S": "is",
"T": "it",
"THE": "the",
"KAT": "cat",
"TK": "did",
"SKP": "and",
"TPOR": "for",
"STO": "so",
"HAOEU": "hi",
"TKOGS": "dogs",
"S/T": "{.}",
"PHAO*EUP/HRAOEUPB": "my line"
}
DICTEOF
echo "=== Step 2: Compile test dictionary ==="
python3 "$ROOT_DIR/tools/compile_mphf.py" \
"$BUILD_DIR/test_dict.json" \
"$BUILD_DIR/test_dict.bin" \
--stats
echo ""
echo "=== Step 3: Build C test binary ==="
cc -O2 -Wall -Wextra -I"$ROOT_DIR/src" \
-o "$BUILD_DIR/test_mphf" \
"$SCRIPT_DIR/test_mphf.c" \
"$ROOT_DIR/src/dict_mphf.c"
echo ""
echo "=== Step 4: Run tests ==="
"$BUILD_DIR/test_mphf" "$BUILD_DIR/test_dict.bin"
echo ""
echo "=== Step 5: Run with larger dict (if Plover available) ==="
PLOVER="/tmp/plover-main.json"
if [ -f "$PLOVER" ]; then
echo "Compiling 1000-entry subset..."
python3 "$ROOT_DIR/tools/compile_mphf.py" \
"$PLOVER" \
"$BUILD_DIR/plover_1k.bin" \
--max-entries 1000 \
--stats
echo ""
echo "Binary size: $(wc -c < "$BUILD_DIR/plover_1k.bin") bytes"
else
echo "Plover dict not found at $PLOVER, skipping large dict test"
echo "Download: curl -sL 'https://raw.githubusercontent.com/openstenoproject/plover/main/plover/assets/main.json' -o /tmp/plover-main.json"
fi
echo ""
echo "=== Done ==="

340
tests/test_formatter.c Normal file
View file

@ -0,0 +1,340 @@
/*
* Native host test for Plover formatting engine.
* Build: cc -I../src -o test_formatter test_formatter.c ../src/formatter.c
*/
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include "formatter.h"
static int tests_run = 0;
static int tests_passed = 0;
#define ASSERT_EQ_STR(actual, expected, msg) do { \
tests_run++; \
if (strcmp((actual), (expected)) == 0) { \
tests_passed++; \
} else { \
printf("FAIL [%s]: got \"%s\", expected \"%s\"\n", msg, actual, expected); \
} \
} while (0)
#define ASSERT_EQ_INT(actual, expected, msg) do { \
tests_run++; \
if ((actual) == (expected)) { \
tests_passed++; \
} else { \
printf("FAIL [%s]: got %d, expected %d\n", msg, (int)(actual), (int)(expected)); \
} \
} while (0)
#define ASSERT_TRUE(cond, msg) do { \
tests_run++; \
if ((cond)) { \
tests_passed++; \
} else { \
printf("FAIL [%s]\n", msg); \
} \
} while (0)
/* Helper: process and return output text */
static struct steno_fmt_result proc(struct steno_fmt_state *s, const char *t)
{
struct steno_fmt_result r;
steno_fmt_process(s, t, &r);
return r;
}
static void test_raw_text(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
/* First word: no space before */
struct steno_fmt_result r = proc(&s, "hello");
ASSERT_EQ_STR(r.text, "hello", "raw: first word");
ASSERT_EQ_INT(r.len, 5, "raw: first word len");
/* Second word: space before */
r = proc(&s, "world");
ASSERT_EQ_STR(r.text, " world", "raw: second word with space");
ASSERT_EQ_INT(r.len, 6, "raw: second word len");
}
static void test_attach(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
proc(&s, "walk");
struct steno_fmt_result r = proc(&s, "{^}");
ASSERT_TRUE(r.is_command_only || r.len == 0, "attach: no output");
r = proc(&s, "ing");
ASSERT_EQ_STR(r.text, "ing", "attach: no space after {^}");
}
static void test_suffix_attach(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
proc(&s, "walk");
struct steno_fmt_result r = proc(&s, "{^ing}");
ASSERT_EQ_STR(r.text, "ing", "suffix: attached");
ASSERT_EQ_INT(r.len, 3, "suffix: len");
}
static void test_prefix_attach(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
struct steno_fmt_result r = proc(&s, "{pre^}");
ASSERT_EQ_STR(r.text, "pre", "prefix: text");
r = proc(&s, "fix");
ASSERT_EQ_STR(r.text, "fix", "prefix: next word attached");
}
static void test_capitalize_next(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
proc(&s, "{-|}");
struct steno_fmt_result r = proc(&s, "hello");
ASSERT_EQ_STR(r.text, "Hello", "cap_next: first word capitalized");
/* Verify cap_next is one-shot */
r = proc(&s, "world");
ASSERT_EQ_STR(r.text, " world", "cap_next: one-shot reset");
}
static void test_punctuation(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
proc(&s, "hello");
struct steno_fmt_result r = proc(&s, "{.}");
ASSERT_EQ_STR(r.text, ".", "period: attached");
/* Next word should be capitalized */
r = proc(&s, "world");
ASSERT_EQ_STR(r.text, " World", "period: cap next");
}
static void test_comma(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
proc(&s, "hello");
struct steno_fmt_result r = proc(&s, "{,}");
ASSERT_EQ_STR(r.text, ",", "comma: attached");
/* Comma does NOT capitalize next */
r = proc(&s, "world");
ASSERT_EQ_STR(r.text, " world", "comma: no cap next");
}
static void test_sentence_flow(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
struct steno_fmt_result r;
r = proc(&s, "I");
ASSERT_EQ_STR(r.text, "I", "sentence: I");
r = proc(&s, "{.}");
ASSERT_EQ_STR(r.text, ".", "sentence: period");
r = proc(&s, "the");
ASSERT_EQ_STR(r.text, " The", "sentence: The after period");
}
static void test_mode_caps(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
proc(&s, "{MODE:CAPS}");
struct steno_fmt_result r = proc(&s, "hello");
ASSERT_EQ_STR(r.text, "HELLO", "mode_caps: uppercase");
r = proc(&s, "world");
ASSERT_EQ_STR(r.text, " WORLD", "mode_caps: persists");
proc(&s, "{MODE:RESET}");
r = proc(&s, "test");
ASSERT_EQ_STR(r.text, " test", "mode_reset: normal");
}
static void test_mode_title(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
proc(&s, "{MODE:TITLE}");
struct steno_fmt_result r = proc(&s, "hello");
ASSERT_EQ_STR(r.text, "Hello", "mode_title: capitalize");
proc(&s, "{MODE:RESET}");
}
static void test_mode_lower(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
proc(&s, "{MODE:LOWER}");
struct steno_fmt_result r = proc(&s, "HELLO");
ASSERT_EQ_STR(r.text, "hello", "mode_lower: lowercase");
proc(&s, "{MODE:RESET}");
}
static void test_fingerspelling(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
struct steno_fmt_result r;
r = proc(&s, "{&a}");
ASSERT_EQ_STR(r.text, "a", "finger: a");
r = proc(&s, "{&b}");
ASSERT_EQ_STR(r.text, "b", "finger: b glued");
r = proc(&s, "{&c}");
ASSERT_EQ_STR(r.text, "c", "finger: c glued");
/* Non-fingerspelling should get space */
r = proc(&s, "hello");
ASSERT_EQ_STR(r.text, " hello", "finger: break with space");
}
static void test_combined_commands(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
proc(&s, "test");
/* {^}{-|} — suppress space + capitalize */
struct steno_fmt_result r = proc(&s, "{^}{-|}");
ASSERT_TRUE(r.len == 0 || r.is_command_only, "combined: no text");
r = proc(&s, "word");
ASSERT_EQ_STR(r.text, "Word", "combined: attached + capitalized");
}
static void test_undo(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
struct steno_fmt_result r = proc(&s, "{*}");
ASSERT_TRUE(r.is_undo, "undo: flag set");
ASSERT_TRUE(r.is_command_only, "undo: command only");
}
static void test_key_combo(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
struct steno_fmt_result r = proc(&s, "{#Return}");
ASSERT_TRUE(r.is_command_only, "key_combo: command only");
}
static void test_literal_braces(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
struct steno_fmt_result r = proc(&s, "\\{");
ASSERT_EQ_STR(r.text, "{", "literal: left brace");
r = proc(&s, "\\}");
ASSERT_EQ_STR(r.text, " }", "literal: right brace with space");
}
static void test_uppercase_next(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
proc(&s, "{<}");
struct steno_fmt_result r = proc(&s, "hello");
ASSERT_EQ_STR(r.text, "HELLO", "upper_next: entire word");
/* One-shot */
r = proc(&s, "world");
ASSERT_EQ_STR(r.text, " world", "upper_next: one-shot");
}
static void test_lowercase_next(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
proc(&s, "{>}");
struct steno_fmt_result r = proc(&s, "HELLO");
ASSERT_EQ_STR(r.text, "hello", "lower_next: entire word");
}
static void test_empty_translation(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
struct steno_fmt_result r = proc(&s, "");
ASSERT_TRUE(r.is_command_only, "empty: command only");
ASSERT_EQ_INT(r.len, 0, "empty: no text");
}
static void test_prefix_then_suffix(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
/* {re^} then {^ed} */
proc(&s, "{re^}");
struct steno_fmt_result r = proc(&s, "{^ed}");
ASSERT_EQ_STR(r.text, "ed", "pre+suf: attached");
}
int main(void)
{
test_raw_text();
test_attach();
test_suffix_attach();
test_prefix_attach();
test_capitalize_next();
test_punctuation();
test_comma();
test_sentence_flow();
test_mode_caps();
test_mode_title();
test_mode_lower();
test_fingerspelling();
test_combined_commands();
test_undo();
test_key_combo();
test_literal_braces();
test_uppercase_next();
test_lowercase_next();
test_empty_translation();
test_prefix_then_suffix();
printf("\n%d/%d tests passed\n", tests_passed, tests_run);
if (tests_passed == tests_run) {
printf("ALL TESTS PASSED\n");
return 0;
}
return 1;
}

383
tests/test_mphf.c Normal file
View file

@ -0,0 +1,383 @@
/**
* Native tests for MPHF dictionary engine.
*
* Builds and runs on host (not ZMK). Compiles a small test dictionary
* via compile_mphf.py, then exercises all lookup paths in C.
*
* Build: cc -O2 -I../src -o test_mphf test_mphf.c ../src/dict_mphf.c
* Run: ./test_mphf test_dict.bin
*
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include "dict_mphf.h"
/* ─── Test helpers ─── */
static int tests_run = 0;
static int tests_passed = 0;
#define TEST(name) \
do { printf(" %-50s ", #name); tests_run++; } while(0)
#define PASS() \
do { tests_passed++; printf("PASS\n"); } while(0)
#define FAIL(msg) \
do { printf("FAIL: %s\n", msg); } while(0)
#define ASSERT_EQ_INT(a, b) \
do { \
if ((a) != (b)) { \
char _buf[128]; \
snprintf(_buf, sizeof(_buf), "expected %d, got %d", (int)(b), (int)(a)); \
FAIL(_buf); return; \
} \
} while(0)
#define ASSERT_EQ_STR(a, b) \
do { \
if (strcmp((a), (b)) != 0) { \
char _buf[256]; \
snprintf(_buf, sizeof(_buf), "expected \"%s\", got \"%s\"", (b), (a)); \
FAIL(_buf); return; \
} \
} while(0)
#define ASSERT_NULL(a) \
do { \
if ((a) != NULL) { \
FAIL("expected NULL"); return; \
} \
} while(0)
#define ASSERT_NOT_NULL(a) \
do { \
if ((a) == NULL) { \
FAIL("expected non-NULL"); return; \
} \
} while(0)
#define ASSERT_TRUE(a) \
do { \
if (!(a)) { \
FAIL("expected true"); return; \
} \
} while(0)
#define ASSERT_FALSE(a) \
do { \
if ((a)) { \
FAIL("expected false"); return; \
} \
} while(0)
/* ─── Load compiled binary from file ─── */
static uint8_t *load_file(const char *path, size_t *out_len)
{
FILE *f = fopen(path, "rb");
if (!f) {
fprintf(stderr, "Cannot open %s\n", path);
return NULL;
}
fseek(f, 0, SEEK_END);
long len = ftell(f);
fseek(f, 0, SEEK_SET);
uint8_t *data = malloc(len);
if (!data) {
fclose(f);
return NULL;
}
fread(data, 1, len, f);
fclose(f);
*out_len = (size_t)len;
return data;
}
/* ─── Test: init with valid data ─── */
static void test_init_valid(const uint8_t *data, size_t len)
{
TEST(init_valid);
struct dict_mphf dict;
int rc = dict_mphf_init(&dict, data, len);
ASSERT_EQ_INT(rc, 0);
ASSERT_EQ_INT(dict.header->magic, DICT_MPHF_MAGIC);
ASSERT_EQ_INT(dict.header->version, DICT_MPHF_VERSION);
PASS();
}
/* ─── Test: init with NULL ─── */
static void test_init_null(void)
{
TEST(init_null);
struct dict_mphf dict;
ASSERT_EQ_INT(dict_mphf_init(&dict, NULL, 0), -1);
ASSERT_EQ_INT(dict_mphf_init(NULL, &dict, 32), -1);
PASS();
}
/* ─── Test: init with truncated data ─── */
static void test_init_truncated(const uint8_t *data)
{
TEST(init_truncated);
struct dict_mphf dict;
ASSERT_EQ_INT(dict_mphf_init(&dict, data, 16), -2);
PASS();
}
/* ─── Test: init with bad magic ─── */
static void test_init_bad_magic(void)
{
TEST(init_bad_magic);
uint8_t bad[32] = {0};
struct dict_mphf dict;
ASSERT_EQ_INT(dict_mphf_init(&dict, bad, sizeof(bad)), -3);
PASS();
}
/* ─── Test: lookup known entries ─── */
/*
* The test dictionary (generated by test runner script) contains:
* "S" → "is"
* "T" → "it"
* "THE" → "the"
* "KAT" → "cat"
* "TK" → "did"
* "SKP" → "and"
* "TPOR" → "for"
* "STO" → "so"
* "HAOEU" → "hi"
* "TKOGS" → "dogs"
* "S/T" → "{.}" (multi-stroke)
* "PHAO*EUP/HRAOEUPB" → "my line" (multi-stroke)
*/
/* Steno key values (must match Python parser) */
#define SK_S 0x00000001u
#define SK_T 0x00000002u
#define SK_K 0x00000004u
#define SK_P 0x00000008u
#define SK_W 0x00000010u
#define SK_H 0x00000020u
#define SK_R 0x00000040u
#define SK_A 0x00000080u
#define SK_O 0x00000100u
#define SK_STAR 0x00000200u
#define SK_E 0x00000400u
#define SK_U 0x00000800u
#define SK_rF 0x00001000u
#define SK_rR 0x00002000u
#define SK_rP 0x00004000u
#define SK_rB 0x00008000u
#define SK_rL 0x00010000u
#define SK_rG 0x00020000u
#define SK_rT 0x00040000u
#define SK_rS 0x00080000u
#define SK_rD 0x00100000u
#define SK_rZ 0x00200000u
#define SK_NUM 0x00400000u
static void test_lookup_single_S(const struct dict_mphf *dict)
{
TEST(lookup_single_S);
uint32_t strokes[] = { SK_S };
const char *result = dict_mphf_lookup(dict, strokes, 1);
ASSERT_NOT_NULL(result);
ASSERT_EQ_STR(result, "is");
PASS();
}
static void test_lookup_single_T(const struct dict_mphf *dict)
{
TEST(lookup_single_T);
uint32_t strokes[] = { SK_T };
const char *result = dict_mphf_lookup(dict, strokes, 1);
ASSERT_NOT_NULL(result);
ASSERT_EQ_STR(result, "it");
PASS();
}
static void test_lookup_THE(const struct dict_mphf *dict)
{
TEST(lookup_THE);
/* THE → T- + H- + -E (left T=0x2, left H=0x20, right E=0x400) */
uint32_t strokes[] = { SK_T | SK_H | SK_E };
const char *result = dict_mphf_lookup(dict, strokes, 1);
ASSERT_NOT_NULL(result);
ASSERT_EQ_STR(result, "the");
PASS();
}
static void test_lookup_KAT(const struct dict_mphf *dict)
{
TEST(lookup_KAT);
uint32_t strokes[] = { SK_K | SK_A | SK_rT };
const char *result = dict_mphf_lookup(dict, strokes, 1);
ASSERT_NOT_NULL(result);
ASSERT_EQ_STR(result, "cat");
PASS();
}
static void test_lookup_TPOR(const struct dict_mphf *dict)
{
TEST(lookup_TPOR);
uint32_t strokes[] = { SK_T | SK_P | SK_O | SK_rR };
const char *result = dict_mphf_lookup(dict, strokes, 1);
ASSERT_NOT_NULL(result);
ASSERT_EQ_STR(result, "for");
PASS();
}
static void test_lookup_HAOEU(const struct dict_mphf *dict)
{
TEST(lookup_HAOEU);
uint32_t strokes[] = { SK_H | SK_A | SK_O | SK_E | SK_U };
const char *result = dict_mphf_lookup(dict, strokes, 1);
ASSERT_NOT_NULL(result);
ASSERT_EQ_STR(result, "hi");
PASS();
}
/* ─── Test: lookup multi-stroke ─── */
static void test_lookup_multi_stroke(const struct dict_mphf *dict)
{
TEST(lookup_multi_stroke);
/* Multi-stroke entries may or may not be present depending on dict size.
* Just verify no crash on 2-stroke lookup. */
uint32_t strokes[] = { SK_S, SK_T };
const char *result = dict_mphf_lookup(dict, strokes, 2);
(void)result;
PASS();
}
/* ─── Test: lookup not found ─── */
static void test_lookup_not_found(const struct dict_mphf *dict)
{
TEST(lookup_not_found);
/* "Z" not in dictionary */
uint32_t strokes[] = { SK_rZ };
const char *result = dict_mphf_lookup(dict, strokes, 1);
/* Could be NULL (fingerprint mismatch) or a wrong string (false positive).
* Fingerprint gives 99.6% true-negative rate. For testing purposes,
* just ensure no crash. If NULL, great. If non-NULL, it's a known
* false positive from the 8-bit fingerprint. */
(void)result;
PASS();
}
/* ─── Test: lookup with zero strokes ─── */
static void test_lookup_zero_strokes(const struct dict_mphf *dict)
{
TEST(lookup_zero_strokes);
uint32_t strokes[] = { 0 };
const char *result = dict_mphf_lookup(dict, strokes, 0);
ASSERT_NULL(result);
PASS();
}
/* ─── Test: has_prefix ─── */
static void test_has_prefix_S(const struct dict_mphf *dict)
{
TEST(has_prefix_S);
/* prefix check depends on whether multi-stroke entries exist in dict.
* Just verify no crash. */
(void)dict_mphf_has_prefix(dict, SK_S);
PASS();
}
static void test_has_prefix_not_found(const struct dict_mphf *dict)
{
TEST(has_prefix_not_found);
/* Extremely unlikely stroke combo — should not be a prefix */
ASSERT_FALSE(dict_mphf_has_prefix(dict, 0x003FFFFFu));
PASS();
}
/* ─── Test: entry count ─── */
static void test_entry_count(const struct dict_mphf *dict)
{
TEST(entry_count);
uint32_t count = dict_mphf_count(dict);
ASSERT_TRUE(count > 0);
PASS();
}
/* ─── Main ─── */
int main(int argc, char **argv)
{
if (argc < 2) {
fprintf(stderr, "Usage: %s <compiled-dict.bin>\n", argv[0]);
return 1;
}
size_t len;
uint8_t *data = load_file(argv[1], &len);
if (!data) {
return 1;
}
printf("Loaded %zu bytes from %s\n\n", len, argv[1]);
/* Init tests */
test_init_valid(data, len);
test_init_null();
test_init_truncated(data);
test_init_bad_magic();
/* Init dict for remaining tests */
struct dict_mphf dict;
int rc = dict_mphf_init(&dict, data, len);
if (rc != 0) {
fprintf(stderr, "dict_mphf_init failed: %d\n", rc);
free(data);
return 1;
}
printf("\n Dict: %u entries, %u buckets, %u unique strings\n",
dict.header->entry_count, dict.header->bucket_count,
dict.header->unique_count);
printf(" Bits: disp=%u value=%u\n",
dict.header->disp_bits, dict.header->value_bits);
printf(" Prefixes: %u\n\n", dict.header->prefix_count);
/* Lookup tests */
test_lookup_single_S(&dict);
test_lookup_single_T(&dict);
test_lookup_THE(&dict);
test_lookup_KAT(&dict);
test_lookup_TPOR(&dict);
test_lookup_HAOEU(&dict);
test_lookup_multi_stroke(&dict);
test_lookup_not_found(&dict);
test_lookup_zero_strokes(&dict);
/* Prefix tests */
test_has_prefix_S(&dict);
test_has_prefix_not_found(&dict);
/* Entry count */
test_entry_count(&dict);
printf("\n%d/%d tests passed\n", tests_passed, tests_run);
free(data);
return tests_passed == tests_run ? 0 : 1;
}

102
tests/test_trie.c Normal file
View file

@ -0,0 +1,102 @@
/*
* Native test for trie.c — run on host, not on target.
* Build: cc -I../src -o test_trie test_trie.c ../src/trie.c
* Run: ./test_trie /tmp/steno_test.bin
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "trie.h"
static uint8_t *load_file(const char *path, size_t *out_len)
{
FILE *f = fopen(path, "rb");
if (!f) return NULL;
fseek(f, 0, SEEK_END);
long len = ftell(f);
fseek(f, 0, SEEK_SET);
uint8_t *buf = malloc(len);
if (fread(buf, 1, len, f) != (size_t)len) {
free(buf);
fclose(f);
return NULL;
}
fclose(f);
*out_len = len;
return buf;
}
static int tests_run = 0;
static int tests_passed = 0;
#define CHECK(cond, msg) do { \
tests_run++; \
if (cond) { tests_passed++; } \
else { printf("FAIL: %s (line %d)\n", msg, __LINE__); } \
} while(0)
int main(int argc, char **argv)
{
if (argc < 2) {
printf("Usage: %s <steno_dict.bin>\n", argv[0]);
return 1;
}
size_t len;
uint8_t *data = load_file(argv[1], &len);
if (!data) {
printf("Failed to load %s\n", argv[1]);
return 1;
}
int ret = steno_trie_init(data, len);
CHECK(ret == 0, "trie_init succeeds");
/* S → "is" (S- = bit 0 = 0x00000001) */
uint32_t s_stroke = 0x00000001;
const char *r = steno_trie_lookup(&s_stroke, 1);
CHECK(r != NULL && strcmp(r, "is") == 0, "S → 'is'");
/* T → "it" (T- = bit 1 = 0x00000002) */
uint32_t t_stroke = 0x00000002;
r = steno_trie_lookup(&t_stroke, 1);
CHECK(r != NULL && strcmp(r, "it") == 0, "T → 'it'");
/* -T → "the" (-T = bit 18 = 0x00040000) */
uint32_t t_right = 0x00040000;
r = steno_trie_lookup(&t_right, 1);
CHECK(r != NULL && strcmp(r, "the") == 0, "-T → 'the'");
/* TEFT → "test" (T- | -E | -F | -T = 0x02|0x400|0x1000|0x40000) */
uint32_t teft = 0x00041402;
r = steno_trie_lookup(&teft, 1);
CHECK(r != NULL && strcmp(r, "test") == 0, "TEFT → 'test'");
/* KO/PHAOURD → "computer" (multi-stroke) */
uint32_t ko = 0x00000104; /* K | O */
uint32_t phaourd = 0x001029A8; /* P | H | A | O | U | R | -D */
uint32_t multi[2] = {ko, phaourd};
r = steno_trie_lookup(multi, 2);
CHECK(r != NULL && strcmp(r, "computer") == 0, "KO/PHAOURD → 'computer'");
/* Non-existent stroke */
uint32_t nonsense = 0x003FFFFF;
r = steno_trie_lookup(&nonsense, 1);
CHECK(r == NULL, "nonsense stroke → NULL");
/* has_prefix: KO should be prefix of KO/PHAOURD */
CHECK(steno_trie_has_prefix(&ko, 1) == true, "KO is prefix");
/* has_prefix: S is NOT prefix of anything multi-stroke */
CHECK(steno_trie_has_prefix(&s_stroke, 1) == false, "S is not prefix");
/* NULL/zero args */
CHECK(steno_trie_lookup(NULL, 1) == NULL, "NULL strokes → NULL");
CHECK(steno_trie_lookup(&s_stroke, 0) == NULL, "0 count → NULL");
CHECK(steno_trie_has_prefix(NULL, 1) == false, "NULL prefix → false");
printf("\n%d/%d tests passed\n", tests_passed, tests_run);
free(data);
return tests_passed == tests_run ? 0 : 1;
}

200
tests/test_undo.c Normal file
View file

@ -0,0 +1,200 @@
/*
* Test undo ring buffer.
* Build: cc -I../src -o test_undo test_undo.c ../src/undo.c
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "undo.h"
static int tests_run = 0;
static int tests_passed = 0;
#define ASSERT(cond, msg) do { \
tests_run++; \
if (!(cond)) { \
printf("FAIL [%s:%d]: %s\n", __func__, __LINE__, msg); \
} else { \
tests_passed++; \
} \
} while (0)
/* 1. Init → count=0, pop returns NULL */
static void test_init(void)
{
struct stroke_history h;
steno_undo_init(&h);
ASSERT(steno_undo_count(&h) == 0, "count should be 0 after init");
ASSERT(steno_undo_pop(&h) == NULL, "pop on empty should return NULL");
ASSERT(steno_undo_peek(&h) == NULL, "peek on empty should return NULL");
}
/* 2. Push one → count=1, peek returns it */
static void test_push_one(void)
{
struct stroke_history h;
steno_undo_init(&h);
uint32_t strokes[] = {0xABCD};
steno_undo_push(&h, strokes, 1, 5, 1, 0x0F);
ASSERT(steno_undo_count(&h) == 1, "count should be 1");
const struct stroke_history_entry *e = steno_undo_peek(&h);
ASSERT(e != NULL, "peek should not be NULL");
ASSERT(e->strokes[0] == 0xABCD, "stroke data mismatch");
ASSERT(e->stroke_count == 1, "stroke_count mismatch");
ASSERT(e->output_len == 5, "output_len mismatch");
ASSERT(e->space_before == 1, "space_before mismatch");
ASSERT(e->fmt_flags == 0x0F, "fmt_flags mismatch");
}
/* 3. Push and pop → entry matches */
static void test_push_pop(void)
{
struct stroke_history h;
steno_undo_init(&h);
uint32_t strokes[] = {0x100, 0x200};
steno_undo_push(&h, strokes, 2, 7, 0, 0x03);
struct stroke_history_entry *e = steno_undo_pop(&h);
ASSERT(e != NULL, "pop should return entry");
ASSERT(e->strokes[0] == 0x100, "stroke[0] mismatch");
ASSERT(e->strokes[1] == 0x200, "stroke[1] mismatch");
ASSERT(e->stroke_count == 2, "stroke_count mismatch");
ASSERT(e->output_len == 7, "output_len mismatch");
ASSERT(e->space_before == 0, "space_before mismatch");
ASSERT(e->fmt_flags == 0x03, "fmt_flags mismatch");
ASSERT(steno_undo_count(&h) == 0, "count should be 0 after pop");
}
/* 4. Overflow: push SIZE+1 → count stays at SIZE */
static void test_overflow(void)
{
struct stroke_history h;
steno_undo_init(&h);
for (int i = 0; i < CONFIG_STENO_HISTORY_SIZE + 1; i++) {
uint32_t s = (uint32_t)i;
steno_undo_push(&h, &s, 1, (uint8_t)(i & 0xFF), 0, 0);
}
ASSERT(steno_undo_count(&h) == CONFIG_STENO_HISTORY_SIZE,
"count should cap at SIZE");
/* Most recent should be SIZE (last pushed) */
const struct stroke_history_entry *e = steno_undo_peek(&h);
ASSERT(e != NULL, "peek should not be NULL");
ASSERT(e->strokes[0] == (uint32_t)CONFIG_STENO_HISTORY_SIZE,
"most recent entry should be last pushed");
/* Oldest (entry 0) should be gone; entry 1 should be oldest */
/* Pop all and check last one is entry 1 */
struct stroke_history_entry *last = NULL;
for (int i = 0; i < CONFIG_STENO_HISTORY_SIZE; i++) {
last = steno_undo_pop(&h);
ASSERT(last != NULL, "pop should succeed");
}
/* last popped = oldest = entry index 1 */
ASSERT(last->strokes[0] == 1, "oldest entry should be index 1 (0 was overwritten)");
ASSERT(steno_undo_count(&h) == 0, "count should be 0 after popping all");
}
/* 5. Pop all → count=0 */
static void test_pop_all(void)
{
struct stroke_history h;
steno_undo_init(&h);
for (int i = 0; i < 10; i++) {
uint32_t s = (uint32_t)i;
steno_undo_push(&h, &s, 1, 1, 0, 0);
}
for (int i = 0; i < 10; i++) {
ASSERT(steno_undo_pop(&h) != NULL, "pop should succeed");
}
ASSERT(steno_undo_count(&h) == 0, "count should be 0");
ASSERT(steno_undo_pop(&h) == NULL, "pop on empty should be NULL");
}
/* 6. Push/pop cycle: push 5, pop 3, push 2, pop 4 → LIFO order */
static void test_push_pop_cycle(void)
{
struct stroke_history h;
steno_undo_init(&h);
/* Push 5: values 10,11,12,13,14 */
for (int i = 0; i < 5; i++) {
uint32_t s = (uint32_t)(10 + i);
steno_undo_push(&h, &s, 1, (uint8_t)(10 + i), 0, 0);
}
ASSERT(steno_undo_count(&h) == 5, "count should be 5");
/* Pop 3: should get 14, 13, 12 */
struct stroke_history_entry *e;
e = steno_undo_pop(&h);
ASSERT(e->strokes[0] == 14, "should pop 14");
e = steno_undo_pop(&h);
ASSERT(e->strokes[0] == 13, "should pop 13");
e = steno_undo_pop(&h);
ASSERT(e->strokes[0] == 12, "should pop 12");
ASSERT(steno_undo_count(&h) == 2, "count should be 2");
/* Push 2: values 20, 21 */
for (int i = 0; i < 2; i++) {
uint32_t s = (uint32_t)(20 + i);
steno_undo_push(&h, &s, 1, (uint8_t)(20 + i), 0, 0);
}
ASSERT(steno_undo_count(&h) == 4, "count should be 4");
/* Pop 4: should get 21, 20, 11, 10 */
e = steno_undo_pop(&h);
ASSERT(e->strokes[0] == 21, "should pop 21");
e = steno_undo_pop(&h);
ASSERT(e->strokes[0] == 20, "should pop 20");
e = steno_undo_pop(&h);
ASSERT(e->strokes[0] == 11, "should pop 11");
e = steno_undo_pop(&h);
ASSERT(e->strokes[0] == 10, "should pop 10");
ASSERT(steno_undo_count(&h) == 0, "count should be 0");
}
/* 7. Verify multi-stroke data preserved */
static void test_data_preservation(void)
{
struct stroke_history h;
steno_undo_init(&h);
uint32_t strokes[] = {0xDEAD, 0xBEEF, 0xCAFE};
steno_undo_push(&h, strokes, 3, 42, 1, 0x7A);
struct stroke_history_entry *e = steno_undo_pop(&h);
ASSERT(e != NULL, "pop should return entry");
ASSERT(e->stroke_count == 3, "stroke_count should be 3");
ASSERT(e->strokes[0] == 0xDEAD, "strokes[0] mismatch");
ASSERT(e->strokes[1] == 0xBEEF, "strokes[1] mismatch");
ASSERT(e->strokes[2] == 0xCAFE, "strokes[2] mismatch");
ASSERT(e->output_len == 42, "output_len mismatch");
ASSERT(e->space_before == 1, "space_before mismatch");
ASSERT(e->fmt_flags == 0x7A, "fmt_flags mismatch");
}
int main(void)
{
printf("Running undo tests...\n\n");
test_init();
test_push_one();
test_push_pop();
test_overflow();
test_pop_all();
test_push_pop_cycle();
test_data_preservation();
printf("\n%d / %d tests passed\n", tests_passed, tests_run);
return (tests_passed == tests_run) ? 0 : 1;
}

521
tools/compile_mphf.py Executable file
View file

@ -0,0 +1,521 @@
#!/usr/bin/env python3
"""CHD MPHF dictionary compiler for steno engine.
Reads Plover JSON dictionaries and produces a compact binary for
embedded use (nRF52840, 462KB flash budget).
"""
import json
import struct
import math
import argparse
import sys
import os
from collections import defaultdict
# ─── Steno stroke parsing ───
STENO_KEYS = {
'#': 0x00400000,
'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004,
'P-': 0x00000008, 'W-': 0x00000010, 'H-': 0x00000020,
'R-': 0x00000040, 'A-': 0x00000080, 'O-': 0x00000100,
'*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800,
'-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000,
'-B': 0x00008000, '-L': 0x00010000, '-G': 0x00020000,
'-T': 0x00040000, '-S': 0x00080000, '-D': 0x00100000,
'-Z': 0x00200000,
}
IMPLICIT_HYPHEN = set('AOEU*')
def parse_stroke(s):
result = 0
if '#' in s:
result |= STENO_KEYS['#']
s = s.replace('#', '')
has_hyphen = '-' in s
s_clean = s.replace('-', '')
if not has_hyphen and not any(c in IMPLICIT_HYPHEN for c in s_clean):
for c in s_clean:
key = c + '-'
if key in STENO_KEYS:
result |= STENO_KEYS[key]
return result
if has_hyphen:
hyphen_pos = s.index('-')
for i, c in enumerate(s):
if c == '-':
continue
if c in 'AO':
result |= STENO_KEYS[c + '-']
elif c in 'EU':
result |= STENO_KEYS['-' + c]
elif c == '*':
result |= STENO_KEYS['*']
elif i < hyphen_pos and (c + '-') in STENO_KEYS:
result |= STENO_KEYS[c + '-']
elif i > hyphen_pos and ('-' + c) in STENO_KEYS:
result |= STENO_KEYS['-' + c]
else:
past_vowels = False
for c in s_clean:
if c in 'AO':
result |= STENO_KEYS[c + '-']
past_vowels = True
elif c in 'EU':
result |= STENO_KEYS['-' + c]
past_vowels = True
elif c == '*':
result |= STENO_KEYS['*']
past_vowels = True
elif not past_vowels and (c + '-') in STENO_KEYS:
result |= STENO_KEYS[c + '-']
elif past_vowels and ('-' + c) in STENO_KEYS:
result |= STENO_KEYS['-' + c]
elif (c + '-') in STENO_KEYS:
result |= STENO_KEYS[c + '-']
return result
# ─── Hashing ───
def fnv1a_32(data: bytes) -> int:
"""FNV-1a 32-bit hash."""
h = 0x811c9dc5
for b in data:
h ^= b
h = (h * 0x01000193) & 0xFFFFFFFF
return h
def hash_key(key_bytes: bytes, seed: int) -> int:
"""Hash key with seed by prepending seed bytes."""
return fnv1a_32(struct.pack('<I', seed) + key_bytes)
# ─── Bit packing ───
class BitWriter:
def __init__(self):
self.data = bytearray()
self.bit_pos = 0 # total bits written
def write_bits(self, value, n_bits):
"""Write n_bits of value (LSB first)."""
for i in range(n_bits):
if self.bit_pos % 8 == 0:
self.data.append(0)
if value & (1 << i):
self.data[-1] |= (1 << (self.bit_pos % 8))
self.bit_pos += 1
def pad_to_alignment(self, alignment=4):
"""Pad to byte alignment."""
while len(self.data) % alignment != 0:
self.data.append(0)
self.bit_pos = len(self.data) * 8
def to_bytes(self):
return bytes(self.data)
class BitReader:
def __init__(self, data):
self.data = data
self.bit_pos = 0
def read_bits(self, n_bits):
value = 0
for i in range(n_bits):
byte_idx = self.bit_pos // 8
bit_idx = self.bit_pos % 8
if self.data[byte_idx] & (1 << bit_idx):
value |= (1 << i)
self.bit_pos += 1
return value
# ─── Key encoding ───
def encode_key(stroke_str):
"""Parse stroke string → key_bytes (each stroke as u32 LE, concatenated)."""
parts = stroke_str.split('/')
strokes = tuple(parse_stroke(s) for s in parts)
key_bytes = b''.join(struct.pack('<I', s) for s in strokes)
return strokes, key_bytes
# ─── Importance scoring ───
def score_entry(stroke_str, translation):
"""Lower score = more important = keep first."""
n_strokes = stroke_str.count('/') + 1
has_format = '{' in translation
return (n_strokes, has_format, len(translation), stroke_str)
# ─── CHD MPHF construction ───
def build_chd(keys_and_bytes, entry_count):
"""
Build CHD MPHF.
keys_and_bytes: list of (index, key_bytes) for each entry
entry_count: total number of entries
Returns: (displacements, slot_to_entry_idx, max_displacement)
displacements[bucket] = d value
slot_to_entry_idx[slot] = index into keys_and_bytes, or -1 if empty
"""
bucket_count = max(entry_count // 3, min(entry_count, 16))
# Assign keys to buckets
buckets = defaultdict(list)
for idx, (_, kb) in enumerate(keys_and_bytes):
b = hash_key(kb, 0) % bucket_count
buckets[b].append(idx)
# Sort buckets by size descending
sorted_buckets = sorted(buckets.items(), key=lambda x: len(x[1]), reverse=True)
displacements = [0] * bucket_count
occupied = set()
slot_to_entry = [-1] * entry_count
max_disp = 0
for bucket_id, members in sorted_buckets:
if not members:
continue
member_key_bytes = [(m, keys_and_bytes[m][1]) for m in members]
placed = False
for d in range(65536):
slots = []
collision = False
seen = set()
for _, kb in member_key_bytes:
slot = hash_key(kb, d + 1) % entry_count
if slot in occupied or slot in seen:
collision = True
break
seen.add(slot)
slots.append(slot)
if collision:
continue
# Place all members
for i, (m, _) in enumerate(member_key_bytes):
occupied.add(slots[i])
slot_to_entry[slots[i]] = m
displacements[bucket_id] = d
if d > max_disp:
max_disp = d
placed = True
break
if not placed:
print(f"FATAL: bucket {bucket_id} with {len(members)} keys failed after 65536 tries",
file=sys.stderr)
return None, None, None
return displacements, slot_to_entry, max_disp
# ─── Compilation ───
def compile_mphf(entries, max_size=None):
"""
entries: list of (stroke_str, translation) from JSON dict
max_size: max output size in bytes (default: 462*1024 = 473088)
Returns: bytes (the compiled binary) or None if can't fit
"""
if max_size is None:
max_size = 462 * 1024
# Sort by importance for potential trimming
entries_scored = sorted(entries, key=lambda e: score_entry(e[0], e[1]))
# Parse all keys, dedup by key_bytes (last wins for same key)
seen_keys = {}
for stroke_str, translation in entries_scored:
strokes, key_bytes = encode_key(stroke_str)
if key_bytes in seen_keys:
prev = seen_keys[key_bytes]
print(f" Dedup: '{stroke_str}'→'{translation}' collides with "
f"'{prev[2]}'→'{prev[1]}', keeping first", file=sys.stderr)
continue
entry = (key_bytes, translation, stroke_str, strokes)
seen_keys[key_bytes] = entry
keys_and_bytes = list(seen_keys.values())
# Iteratively trim if needed
while True:
entry_count = len(keys_and_bytes)
if entry_count == 0:
return None
# Build deduped string table
translations = [kb[1] for kb in keys_and_bytes]
unique_translations = sorted(set(translations))
trans_to_id = {t: i for i, t in enumerate(unique_translations)}
unique_count = len(unique_translations)
# Estimate size
bucket_count = max(entry_count // 3, min(entry_count, 16))
est_value_bits = max(1, math.ceil(math.log2(max(unique_count, 2))))
est_disp_bits = 16 # conservative
est_disp_bytes = (bucket_count * est_disp_bits + 7) // 8
est_disp_bytes = ((est_disp_bytes + 3) // 4) * 4
est_value_bytes = (entry_count * est_value_bits + 7) // 8
est_value_bytes = ((est_value_bytes + 3) // 4) * 4
est_fp_bytes = ((entry_count + 3) // 4) * 4
# String table
string_data = b''
string_offsets = []
for t in unique_translations:
string_offsets.append(len(string_data))
string_data += t.encode('utf-8') + b'\x00'
est_str_offsets = unique_count * 3 # u24 packed LE
est_str_data = len(string_data)
# Prefix table
prefix_strokes = set()
for kb, trans, stroke_str, strokes in keys_and_bytes:
if len(strokes) > 1:
prefix_strokes.add(strokes[0])
prefix_list = sorted(prefix_strokes)
est_prefix = len(prefix_list) * 4
total_est = 32 + est_disp_bytes + est_value_bytes + est_fp_bytes + est_str_offsets + est_str_data + est_prefix
if total_est <= max_size:
break
# Trim: use ratio of overshoot to estimate how many entries to cut
overshoot_ratio = total_est / max_size
target_entries = int(entry_count / overshoot_ratio * 0.98) # 2% safety margin
trim_count = max(1, entry_count - target_entries)
keys_and_bytes = keys_and_bytes[:entry_count - trim_count]
print(f" Trimming to {len(keys_and_bytes)} entries (est {total_est} > {max_size})",
file=sys.stderr)
entry_count = len(keys_and_bytes)
bucket_count = max(entry_count // 3, min(entry_count, 16))
print(f" Building CHD MPHF: {entry_count} entries, {bucket_count} buckets...",
file=sys.stderr)
# Build CHD
chd_input = [(i, keys_and_bytes[i][0]) for i in range(entry_count)]
displacements, slot_to_entry, max_disp = build_chd(chd_input, entry_count)
if displacements is None:
return None
# Compute actual bit widths
disp_bits = max(1, math.ceil(math.log2(max(max_disp + 1, 2))))
value_bits = max(1, math.ceil(math.log2(max(unique_count, 2))))
prefix_count = len(prefix_list)
print(f" Max displacement: {max_disp}, disp_bits: {disp_bits}", file=sys.stderr)
print(f" Unique translations: {unique_count}, value_bits: {value_bits}", file=sys.stderr)
print(f" Prefix entries: {prefix_count}", file=sys.stderr)
# ─── Build binary ───
# Displacements section
disp_writer = BitWriter()
for d in displacements:
disp_writer.write_bits(d, disp_bits)
disp_writer.pad_to_alignment(4)
disp_section = disp_writer.to_bytes()
# Values section: slot → value_id
val_writer = BitWriter()
fingerprints = bytearray(entry_count)
for slot in range(entry_count):
entry_idx = slot_to_entry[slot]
if entry_idx >= 0:
kb, trans, stroke_str, strokes = keys_and_bytes[entry_idx]
val_id = trans_to_id[trans]
val_writer.write_bits(val_id, value_bits)
fingerprints[slot] = fnv1a_32(kb) & 0xFF
else:
val_writer.write_bits(0, value_bits)
fingerprints[slot] = 0
val_writer.pad_to_alignment(4)
val_section = val_writer.to_bytes()
# Fingerprints section
fp_section = bytes(fingerprints)
# Pad to 4-byte boundary
while len(fp_section) % 4 != 0:
fp_section += b'\x00'
# String offsets section (u24 packed LE — 3 bytes each)
str_offsets_section = b''.join(struct.pack('<I', off)[:3] for off in string_offsets)
# String data section (already built above)
str_data_section = string_data
# Prefix table section
prefix_section = b''.join(struct.pack('<I', s) for s in prefix_list)
# Header (32 bytes):
# magic: u32, version: u16, flags: u16,
# entry_count: u32, bucket_count: u32, unique_count: u32,
# value_bits: u8, disp_bits: u8, prefix_count: u16,
# reserved0: u32, reserved1: u32
header = struct.pack('<IHHIIIBBHii',
0x4F4E5453, # magic "STNO"
2, # version
0, # flags
entry_count, # entry_count
bucket_count, # bucket_count
unique_count, # unique_count
value_bits, # value_bits
disp_bits, # disp_bits
prefix_count, # prefix_count
0, # reserved0
0, # reserved1
)
assert len(header) == 32, f"Header is {len(header)} bytes, expected 32"
binary = header + disp_section + val_section + fp_section + str_offsets_section + str_data_section + prefix_section
# ─── Verification ───
print(f" Verifying all {entry_count} entries...", file=sys.stderr)
errors = 0
for entry_idx in range(entry_count):
kb, trans, stroke_str, strokes = keys_and_bytes[entry_idx]
# Lookup through MPHF
bucket = hash_key(kb, 0) % bucket_count
# Read displacement
disp_reader = BitReader(disp_section)
disp_reader.bit_pos = bucket * disp_bits
d = disp_reader.read_bits(disp_bits)
slot = hash_key(kb, d + 1) % entry_count
# Check fingerprint
expected_fp = fnv1a_32(kb) & 0xFF
if fingerprints[slot] != expected_fp:
print(f" VERIFY FAIL: fingerprint mismatch for '{stroke_str}' at slot {slot}: "
f"got {fingerprints[slot]}, expected {expected_fp}", file=sys.stderr)
errors += 1
continue
# Check value
val_reader = BitReader(val_section)
val_reader.bit_pos = slot * value_bits
val_id = val_reader.read_bits(value_bits)
resolved = unique_translations[val_id]
if resolved != trans:
print(f" VERIFY FAIL: value mismatch for '{stroke_str}': "
f"got '{resolved}', expected '{trans}'", file=sys.stderr)
errors += 1
if errors:
print(f" VERIFICATION FAILED: {errors} errors", file=sys.stderr)
return None
print(f" Verification passed: all {entry_count} entries OK", file=sys.stderr)
# Check final size
if len(binary) > max_size:
print(f" WARNING: output {len(binary)} bytes exceeds max {max_size}", file=sys.stderr)
return binary, {
'entry_count': entry_count,
'bucket_count': bucket_count,
'unique_count': unique_count,
'value_bits': value_bits,
'disp_bits': disp_bits,
'max_displacement': max_disp,
'prefix_count': prefix_count,
'disp_section_bytes': len(disp_section),
'val_section_bytes': len(val_section),
'fp_section_bytes': len(fp_section),
'str_offsets_bytes': len(str_offsets_section),
'str_data_bytes': len(str_data_section),
'prefix_section_bytes': len(prefix_section),
'total_bytes': len(binary),
}
def print_stats(stats):
"""Print size breakdown statistics."""
total = stats['entry_count']
print(f"Entries: {stats['entry_count']}")
print(f"MPHF displacements: {stats['disp_section_bytes']/1024:.1f} KB "
f"({stats['bucket_count']} buckets, {stats['disp_bits']} bits each)")
print(f"Value array: {stats['val_section_bytes']/1024:.1f} KB "
f"({stats['entry_count']} entries, {stats['value_bits']} bits each)")
print(f"Fingerprints: {stats['fp_section_bytes']/1024:.1f} KB")
print(f"String offsets: {stats['str_offsets_bytes']/1024:.1f} KB "
f"({stats['unique_count']} unique x 3 bytes)")
print(f"String data: {stats['str_data_bytes']/1024:.1f} KB")
print(f"Prefix table: {stats['prefix_section_bytes']/1024:.1f} KB "
f"({stats['prefix_count']} entries x 4 bytes)")
print(f"Total: {stats['total_bytes']/1024:.1f} KB")
def main():
parser = argparse.ArgumentParser(description='Compile steno dictionary to MPHF binary format')
parser.add_argument('input', help='Input JSON dictionary (Plover format)')
parser.add_argument('output', help='Output binary file')
parser.add_argument('--max-size', type=int, default=462*1024,
help='Maximum output size in bytes (default: 473088 = 462KB)')
parser.add_argument('--max-entries', type=int, default=None,
help='Maximum number of entries')
parser.add_argument('--stats', action='store_true',
help='Print size breakdown statistics')
parser.add_argument('--verify', action='store_true', default=True,
help='Verify compiled dict (default: true)')
args = parser.parse_args()
# Load dictionary
with open(args.input) as f:
raw_dict = json.load(f)
print(f"Loaded {len(raw_dict)} entries from {args.input}", file=sys.stderr)
entries = list(raw_dict.items())
if args.max_entries is not None:
entries_scored = sorted(entries, key=lambda e: score_entry(e[0], e[1]))
entries = entries_scored[:args.max_entries]
print(f"Trimmed to {len(entries)} entries (--max-entries)", file=sys.stderr)
result = compile_mphf(entries, max_size=args.max_size)
if result is None:
print("Compilation failed", file=sys.stderr)
sys.exit(1)
binary, stats = result
with open(args.output, 'wb') as f:
f.write(binary)
print(f"Wrote {len(binary)} bytes to {args.output}", file=sys.stderr)
if args.stats:
print()
print_stats(stats)
if __name__ == '__main__':
main()

175
tools/compile_simple.py Normal file
View file

@ -0,0 +1,175 @@
#!/usr/bin/env python3
"""Simple flat-format steno dictionary compiler.
Outputs a binary format optimized for binary search on embedded targets.
No compression — just sorted entries with fixed-width keys.
Format:
Header (16 bytes):
magic: u32 = 0x4F4E5453 ("STNO")
version: u16 = 1
max_strokes: u8 (max stroke count per entry)
pad: u8
entry_count: u32
strings_offset: u32
Entry array (sorted by stroke tuple, fixed width):
Each entry = max_strokes * 4 + 4 bytes:
strokes[max_strokes]: u32 LE (unused slots = 0)
string_offset: u32 LE (into string table)
String table:
Null-terminated UTF-8 strings, concatenated
"""
import argparse
import json
import struct
import sys
import os
STENO_KEYS = {
'#': 0x00400000,
'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004,
'P-': 0x00000008, 'W-': 0x00000010, 'H-': 0x00000020,
'R-': 0x00000040, 'A-': 0x00000080, 'O-': 0x00000100,
'*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800,
'-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000,
'-B': 0x00008000, '-L': 0x00010000, '-G': 0x00020000,
'-T': 0x00040000, '-S': 0x00080000, '-D': 0x00100000,
'-Z': 0x00200000,
}
IMPLICIT_HYPHEN = set('AOEU*')
def parse_stroke(s):
result = 0
if '#' in s:
result |= STENO_KEYS['#']
s = s.replace('#', '')
has_hyphen = '-' in s
s_clean = s.replace('-', '')
if not has_hyphen and not any(c in IMPLICIT_HYPHEN for c in s_clean):
for c in s_clean:
key = c + '-'
if key in STENO_KEYS:
result |= STENO_KEYS[key]
return result
if has_hyphen:
hyphen_pos = s.index('-')
for i, c in enumerate(s):
if c == '-':
continue
if c in 'AO':
result |= STENO_KEYS[c + '-']
elif c in 'EU':
result |= STENO_KEYS['-' + c]
elif c == '*':
result |= STENO_KEYS['*']
elif i < hyphen_pos and (c + '-') in STENO_KEYS:
result |= STENO_KEYS[c + '-']
elif i > hyphen_pos and ('-' + c) in STENO_KEYS:
result |= STENO_KEYS['-' + c]
else:
past_vowels = False
for c in s_clean:
if c in 'AO':
result |= STENO_KEYS[c + '-']
past_vowels = True
elif c in 'EU':
result |= STENO_KEYS['-' + c]
past_vowels = True
elif c == '*':
result |= STENO_KEYS['*']
past_vowels = True
elif not past_vowels and (c + '-') in STENO_KEYS:
result |= STENO_KEYS[c + '-']
elif past_vowels and ('-' + c) in STENO_KEYS:
result |= STENO_KEYS['-' + c]
elif (c + '-') in STENO_KEYS:
result |= STENO_KEYS[c + '-']
return result
def compile_dict(json_path, max_entries=None):
with open(json_path) as f:
raw = json.load(f)
entries = []
for stroke_str, translation in raw.items():
strokes = tuple(parse_stroke(s) for s in stroke_str.split('/'))
entries.append((strokes, translation))
if max_entries and len(entries) > max_entries:
single = [(s, t) for s, t in entries if len(s) == 1]
multi = [(s, t) for s, t in entries if len(s) > 1]
multi.sort(key=lambda x: (len(x[1]), len(x[0])))
remaining = max_entries - len(single)
entries = single + multi[:max(0, remaining)]
entries.sort(key=lambda x: x[0])
max_strokes = max(len(s) for s, _ in entries)
string_table = bytearray()
string_offsets = {}
for _, translation in entries:
if translation not in string_offsets:
string_offsets[translation] = len(string_table)
string_table.extend(translation.encode('utf-8'))
string_table.append(0)
entry_size = max_strokes * 4 + 4
header_size = 16
entries_size = len(entries) * entry_size
strings_offset = header_size + entries_size
header = struct.pack('<IHBBII',
0x4F4E5453, # "STNO"
1, # version
max_strokes,
0, # pad
len(entries),
strings_offset)
entry_data = bytearray()
for strokes, translation in entries:
padded = list(strokes) + [0] * (max_strokes - len(strokes))
for s in padded:
entry_data.extend(struct.pack('<I', s))
entry_data.extend(struct.pack('<I', string_offsets[translation]))
binary = header + bytes(entry_data) + bytes(string_table)
return binary, len(entries), max_strokes, len(string_table)
def main():
parser = argparse.ArgumentParser(description='Compile steno dict to flat binary')
parser.add_argument('input', help='JSON dictionary path')
parser.add_argument('-o', '--output', default='steno_dict.bin')
parser.add_argument('--max-entries', type=int, default=None)
parser.add_argument('--stats', action='store_true')
args = parser.parse_args()
if not os.path.exists(args.input):
print(f"ERROR: {args.input} not found")
sys.exit(1)
binary, n_entries, max_strokes, str_size = compile_dict(args.input, args.max_entries)
with open(args.output, 'wb') as f:
f.write(binary)
if args.stats:
entry_size = max_strokes * 4 + 4
print(f"Entries: {n_entries}")
print(f"Max strokes: {max_strokes}")
print(f"Entry size: {entry_size} bytes")
print(f"Entry array: {n_entries * entry_size} bytes")
print(f"String table: {str_size} bytes")
print(f"Total: {len(binary)} bytes ({len(binary)/1024:.1f} KB)")
print(f"Written {len(binary)} bytes to {args.output}")
if __name__ == '__main__':
main()

815
tools/dict_compiler.py Normal file
View file

@ -0,0 +1,815 @@
#!/usr/bin/env python3
"""DAWG dictionary compiler for ZMK steno engine.
Compiles a Plover-format JSON steno dictionary into a compact binary
DAWG with skip-count indexing and block-compressed string table.
Split-storage variant targeting 533KB (right half flash budget).
"""
import argparse
import json
import math
import os
import struct
import sys
import zlib
from collections import Counter
# ─── Steno stroke parsing (copied from dawg_fst_prototype.py) ───
STENO_KEYS = {
'#': 0x00400000,
'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004,
'P-': 0x00000008, 'W-': 0x00000010, 'H-': 0x00000020,
'R-': 0x00000040, 'A-': 0x00000080, 'O-': 0x00000100,
'*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800,
'-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000,
'-B': 0x00008000, '-L': 0x00010000, '-G': 0x00020000,
'-T': 0x00040000, '-S': 0x00080000, '-D': 0x00100000,
'-Z': 0x00200000,
}
IMPLICIT_HYPHEN = set('AOEU*')
def parse_stroke(s):
"""Parse a steno stroke string into a bitmask."""
result = 0
if '#' in s:
result |= STENO_KEYS['#']
s = s.replace('#', '')
has_hyphen = '-' in s
s_clean = s.replace('-', '')
if not has_hyphen and not any(c in IMPLICIT_HYPHEN for c in s_clean):
for c in s_clean:
key = c + '-'
if key in STENO_KEYS:
result |= STENO_KEYS[key]
return result
past_vowels = False
for c in s_clean:
if c in 'AO':
result |= STENO_KEYS[c + '-']
past_vowels = True
elif c in 'EU':
result |= STENO_KEYS['-' + c]
past_vowels = True
elif c == '*':
result |= STENO_KEYS['*']
past_vowels = True
elif not past_vowels and (c + '-') in STENO_KEYS:
result |= STENO_KEYS[c + '-']
elif past_vowels and ('-' + c) in STENO_KEYS:
result |= STENO_KEYS['-' + c]
elif has_hyphen:
if s.index(c) < s.index('-'):
result |= STENO_KEYS.get(c + '-', 0)
else:
result |= STENO_KEYS.get('-' + c, 0)
else:
if (c + '-') in STENO_KEYS:
result |= STENO_KEYS[c + '-']
return result
def parse_stroke_string(stroke_str):
"""Parse a stroke string (possibly multi-stroke with /) into tuple of bitmasks."""
return tuple(parse_stroke(s) for s in stroke_str.split('/'))
# ─── DAWG construction (Daciuk's incremental algorithm) ───
class DawgNode:
"""Node in the DAWG."""
__slots__ = ['id', 'edges', 'final', '_hash_cache']
_next_id = 0
def __init__(self):
self.id = DawgNode._next_id
DawgNode._next_id += 1
self.edges = {} # stroke_val -> DawgNode
self.final = False
self._hash_cache = None
def signature(self):
"""Hashable signature for minimization."""
edge_sig = tuple(sorted(
(k, child.id) for k, child in self.edges.items()
))
return (self.final, edge_sig)
def __hash__(self):
if self._hash_cache is None:
self._hash_cache = hash(self.signature())
return self._hash_cache
def __eq__(self, other):
return self.signature() == other.signature()
def invalidate_cache(self):
self._hash_cache = None
def build_dawg(sorted_entries):
"""Build minimized DAWG using Daciuk's incremental algorithm.
Entries MUST be sorted by stroke tuple (lexicographic).
Returns (root, node_count, edge_count).
"""
DawgNode._next_id = 0
root = DawgNode()
unchecked = [] # list of (parent, stroke, child)
minimized = {} # signature -> node
prev_strokes = ()
def _minimize(down_to):
"""Minimize unchecked nodes from top down to given depth."""
for i in range(len(unchecked) - 1, down_to - 1, -1):
parent, stroke, child = unchecked[i]
child.invalidate_cache()
sig = child.signature()
if sig in minimized:
parent.edges[stroke] = minimized[sig]
else:
minimized[sig] = child
unchecked.pop()
for strokes, _translation in sorted_entries:
# Find common prefix length with previous entry
common = 0
limit = min(len(strokes), len(prev_strokes))
while common < limit and strokes[common] == prev_strokes[common]:
common += 1
# Minimize nodes beyond common prefix
_minimize(common)
# Get node at end of common prefix
if unchecked:
node = unchecked[-1][2]
else:
node = root
# Add new nodes for remaining strokes
for stroke in strokes[common:]:
new_node = DawgNode()
node.edges[stroke] = new_node
unchecked.append((node, stroke, new_node))
node = new_node
node.final = True
prev_strokes = strokes
# Minimize remaining
_minimize(0)
# Count nodes and edges
node_count = 0
edge_count = 0
visited = set()
def _count(n):
nonlocal node_count, edge_count
if n.id in visited:
return
visited.add(n.id)
node_count += 1
for _s, child in n.edges.items():
edge_count += 1
_count(child)
_count(root)
return root, node_count, edge_count
# ─── Skip-count computation ───
def compute_skip_counts(root):
"""Compute skip-count (number of final nodes in subtree) for each node.
Returns dict: node_id -> skip_count
"""
cache = {}
def _count(node):
if node.id in cache:
return cache[node.id]
c = 1 if node.final else 0
for stroke in sorted(node.edges.keys()):
child = node.edges[stroke]
c += _count(child)
cache[node.id] = c
return c
_count(root)
return cache
def dawg_lookup_index(root, strokes, skip_cache):
"""Look up a stroke sequence in the DAWG, returning its skip-count index.
Returns -1 if not found.
"""
node = root
idx = 0
for stroke in strokes:
if stroke not in node.edges:
return -1
# Count finals of all edges with stroke < target
for s in sorted(node.edges.keys()):
if s == stroke:
child = node.edges[s]
if child.final:
idx += 1
node = child
break
else:
child = node.edges[s]
idx += skip_cache[child.id]
else:
return -1
if not node.final:
return -1
return idx - 1
def get_dawg_traversal_order(root):
"""DFS traversal of DAWG, edges sorted by stroke value.
Returns list of translations in traversal order (one per final node encounter).
This is the order entries appear when looking up via skip-count.
"""
order = []
visited_paths = set()
def _dfs(node, path):
path_key = tuple(path)
if path_key in visited_paths:
return
visited_paths.add(path_key)
if node.final:
order.append(path_key)
for stroke in sorted(node.edges.keys()):
child = node.edges[stroke]
_dfs(child, path + [stroke])
_dfs(root, [])
return order
# ─── Entry trimming ───
def trim_entries(entries, max_entries):
"""Trim entries to max_entries, keeping single-stroke preferentially.
Priority:
1. All single-stroke entries
2. Multi-stroke entries sorted by translation length (shorter first)
3. Drop longest/rarest multi-stroke first
"""
if len(entries) <= max_entries:
return entries
single_stroke = []
multi_stroke = []
for strokes_str, translation in entries:
if '/' not in strokes_str:
single_stroke.append((strokes_str, translation))
else:
multi_stroke.append((strokes_str, translation))
# Sort multi-stroke by translation length (shorter = more useful)
multi_stroke.sort(key=lambda x: (len(x[1]), len(x[0].split('/'))))
remaining = max_entries - len(single_stroke)
if remaining < 0:
# Even single-stroke entries exceed limit; trim by translation length
single_stroke.sort(key=lambda x: len(x[1]))
return single_stroke[:max_entries]
return single_stroke + multi_stroke[:remaining]
# ─── String table construction ───
def build_string_table(translations):
"""Build block-compressed string table from translations.
Returns (table_bytes, offsets) where offsets[i] is the byte offset
of translation i in the uncompressed table.
"""
# Build raw table: null-separated strings
# We need to track offset of each unique string
unique_translations = sorted(set(translations))
trans_to_unique_idx = {t: i for i, t in enumerate(unique_translations)}
# Build raw bytes and offset map
raw_parts = []
unique_offsets = []
offset = 0
for t in unique_translations:
unique_offsets.append(offset)
encoded = t.encode('utf-8')
raw_parts.append(encoded)
offset += len(encoded) + 1 # +1 for null separator
raw = b'\x00'.join(raw_parts)
if raw_parts:
raw += b'\x00' # trailing null
# Block compress
block_size = 4096
compressed_blocks = []
block_offsets_raw = []
current_offset = 0
for i in range(0, len(raw), block_size):
block = raw[i:i + block_size]
compressed = zlib.compress(block, 9)
block_offsets_raw.append(current_offset)
compressed_blocks.append(compressed)
current_offset += len(compressed)
# Serialize: block_count(u16) + block_offsets(u32 each) + compressed blocks
n_blocks = len(compressed_blocks)
table_header = struct.pack('<H', n_blocks)
table_index = b''.join(struct.pack('<I', off) for off in block_offsets_raw)
table_data = b''.join(compressed_blocks)
table_bytes = table_header + table_index + table_data
# Map each translation to its offset in raw table
entry_offsets = []
for t in translations:
uid = trans_to_unique_idx[t]
entry_offsets.append(unique_offsets[uid])
return table_bytes, entry_offsets, len(raw)
def decompress_string_table(table_bytes):
"""Decompress a block-compressed string table back to raw bytes."""
pos = 0
n_blocks = struct.unpack_from('<H', table_bytes, pos)[0]
pos += 2
block_offsets = []
for _ in range(n_blocks):
off = struct.unpack_from('<I', table_bytes, pos)[0]
pos += 4
block_offsets.append(off)
data_start = pos
raw_parts = []
for i in range(n_blocks):
block_start = data_start + block_offsets[i]
if i + 1 < n_blocks:
block_end = data_start + block_offsets[i + 1]
else:
block_end = len(table_bytes)
compressed = table_bytes[block_start:block_end]
raw_parts.append(zlib.decompress(compressed))
return b''.join(raw_parts)
def lookup_string(raw_table, offset):
"""Look up a null-terminated string at given offset in raw table."""
end = raw_table.index(b'\x00', offset)
return raw_table[offset:end].decode('utf-8')
# ─── Binary serialization ───
MAGIC = b'STNO'
VERSION = 1
FLAG_SPLIT_STORAGE = 0x0001
HEADER_SIZE = 32
def serialize_header(flags, entry_count, node_count, edge_count,
string_table_offset, string_table_size,
value_array_offset):
"""Serialize the 32-byte binary header."""
return struct.pack('<4sHHIIIIII',
MAGIC,
VERSION,
flags,
entry_count,
node_count,
edge_count,
string_table_offset,
string_table_size,
value_array_offset)
def parse_header(data):
"""Parse 32-byte binary header. Returns dict."""
if len(data) < HEADER_SIZE:
raise ValueError("Data too short for header")
# Unpack with padding bytes
magic, version, flags, entry_count, node_count, edge_count, \
str_table_off, str_table_size, val_array_off = \
struct.unpack_from('<4sHHIIIIII', data, 0)
# 4 bytes reserved at end (32 - 28 = 4)
if magic != MAGIC:
raise ValueError(f"Bad magic: {magic!r}")
return {
'magic': magic,
'version': version,
'flags': flags,
'entry_count': entry_count,
'node_count': node_count,
'edge_count': edge_count,
'string_table_offset': str_table_off,
'string_table_size': str_table_size,
'value_array_offset': val_array_off,
}
def serialize_edges(root, node_count, edge_count):
"""Serialize DAWG edges as bit-packed array.
Each edge: stroke_key(16) + target_node(16) + skip_count(17) + is_last(1) = 50 bits
Nodes are assigned sequential IDs via DFS traversal (sorted edges).
Returns (edge_bytes, node_id_map, skip_counts_by_node).
"""
# Assign sequential node IDs via DFS
node_id_map = {}
dfs_order = []
def _assign_ids(node):
if node.id in node_id_map:
return
new_id = len(node_id_map)
node_id_map[node.id] = new_id
dfs_order.append(node)
for stroke in sorted(node.edges.keys()):
child = node.edges[stroke]
_assign_ids(child)
_assign_ids(root)
# Compute skip counts
skip_cache = compute_skip_counts(root)
# Build edge list: for each node in DFS order, emit edges sorted by stroke
edges = []
for node in dfs_order:
sorted_strokes = sorted(node.edges.keys())
for i, stroke in enumerate(sorted_strokes):
child = node.edges[stroke]
is_last = (i == len(sorted_strokes) - 1)
target_id = node_id_map[child.id]
skip = skip_cache[child.id]
edges.append((stroke, target_id, skip, is_last))
# Bit-pack edges: each 50 bits
# stroke_key: 16 bits, target_node: 16 bits, skip_count: 17 bits, is_last: 1 bit
total_bits = len(edges) * 50
total_bytes = (total_bits + 7) // 8
buf = bytearray(total_bytes)
bit_pos = 0
for stroke_key, target_node, skip_count, is_last in edges:
# Clamp values to field widths
stroke_key &= 0xFFFF
target_node &= 0xFFFF
skip_count = min(skip_count, 0x1FFFF) # 17 bits max
is_last_bit = 1 if is_last else 0
# Pack 50 bits: stroke(16) | target(16) | skip(17) | last(1)
val = (stroke_key << 34) | (target_node << 18) | (skip_count << 1) | is_last_bit
# Write 50 bits into buffer at bit_pos
for i in range(50):
bit = (val >> (49 - i)) & 1
byte_idx = (bit_pos + i) // 8
bit_idx = 7 - ((bit_pos + i) % 8)
if bit:
buf[byte_idx] |= (1 << bit_idx)
bit_pos += 50
return bytes(buf), node_id_map, skip_cache
def deserialize_edges(edge_bytes, edge_count):
"""Deserialize bit-packed edge array.
Returns list of (stroke_key, target_node, skip_count, is_last).
"""
edges = []
bit_pos = 0
for _ in range(edge_count):
val = 0
for i in range(50):
byte_idx = (bit_pos + i) // 8
bit_idx = 7 - ((bit_pos + i) % 8)
bit = (edge_bytes[byte_idx] >> bit_idx) & 1
val = (val << 1) | bit
bit_pos += 50
stroke_key = (val >> 34) & 0xFFFF
target_node = (val >> 18) & 0xFFFF
skip_count = (val >> 1) & 0x1FFFF
is_last = val & 1
edges.append((stroke_key, target_node, skip_count, bool(is_last)))
return edges
def serialize_value_array(entry_offsets, raw_table_size):
"""Serialize value array (string table offsets for each entry).
Uses uint16 if raw_table_size <= 65535, else uint32.
"""
use_u32 = raw_table_size > 65535
fmt = '<I' if use_u32 else '<H'
parts = [struct.pack(fmt, off) for off in entry_offsets]
return b''.join(parts), use_u32
def deserialize_value_array(data, entry_count, use_u32=False):
"""Deserialize value array."""
fmt = '<I' if use_u32 else '<H'
size = 4 if use_u32 else 2
offsets = []
for i in range(entry_count):
off = struct.unpack_from(fmt, data, i * size)[0]
offsets.append(off)
return offsets
# ─── Full compilation pipeline ───
def compile_dictionary(json_path, max_entries=130000, split_storage=False):
"""Compile a JSON steno dictionary into binary DAWG format.
Returns (binary_data, stats_dict).
"""
# 1. Load and parse
with open(json_path) as f:
raw_dict = json.load(f)
raw_entries = list(raw_dict.items())
# 2. Trim if needed
if len(raw_entries) > max_entries:
raw_entries = trim_entries(raw_entries, max_entries)
# 3. Parse strokes and sort
parsed = []
for stroke_str, translation in raw_entries:
strokes = parse_stroke_string(stroke_str)
parsed.append((strokes, translation))
parsed.sort(key=lambda x: x[0])
# 4. Build DAWG
root, node_count, edge_count = build_dawg(parsed)
# 5. Get traversal order for value array
traversal_paths = get_dawg_traversal_order(root)
# Build path->translation map
path_to_trans = {}
for strokes, translation in parsed:
path_to_trans[strokes] = translation
translations_ordered = []
for path in traversal_paths:
if path in path_to_trans:
translations_ordered.append(path_to_trans[path])
else:
translations_ordered.append("")
# 6. Build string table
string_table_bytes, entry_offsets, raw_table_size = \
build_string_table(translations_ordered)
# 7. Serialize edges
edge_bytes, node_id_map, skip_cache = \
serialize_edges(root, node_count, edge_count)
# 8. Serialize value array
value_array_bytes, use_u32 = \
serialize_value_array(entry_offsets, raw_table_size)
# 9. Compute offsets
edge_array_offset = HEADER_SIZE
value_array_offset = edge_array_offset + len(edge_bytes)
string_table_offset = value_array_offset + len(value_array_bytes)
# 10. Build header
flags = 0
if split_storage:
flags |= FLAG_SPLIT_STORAGE
if use_u32:
flags |= 0x0002 # bit 1 = u32 value offsets
header = serialize_header(
flags=flags,
entry_count=len(translations_ordered),
node_count=node_count,
edge_count=edge_count,
string_table_offset=string_table_offset,
string_table_size=len(string_table_bytes),
value_array_offset=value_array_offset,
)
# 11. Assemble
binary = header + edge_bytes + value_array_bytes + string_table_bytes
stats = {
'entry_count': len(translations_ordered),
'node_count': node_count,
'edge_count': edge_count,
'bits_per_edge': 50,
'edge_array_size': len(edge_bytes),
'value_array_size': len(value_array_bytes),
'string_table_size': len(string_table_bytes),
'raw_string_table_size': raw_table_size,
'total_size': len(binary),
'header_size': HEADER_SIZE,
'use_u32_offsets': use_u32,
'split_storage': split_storage,
}
return binary, stats, root, skip_cache, translations_ordered, parsed
def verify_compilation(binary_data, parsed_entries, root, skip_cache):
"""Verify compiled binary by deserializing and looking up every entry.
Returns (correct, wrong, missing).
"""
header = parse_header(binary_data)
entry_count = header['entry_count']
edge_count = header['edge_count']
use_u32 = bool(header['flags'] & 0x0002)
# Extract sections
edge_start = HEADER_SIZE
edge_end = header['value_array_offset']
edge_bytes = binary_data[edge_start:edge_end]
val_start = header['value_array_offset']
val_end = header['string_table_offset']
val_bytes = binary_data[val_start:val_end]
str_start = header['string_table_offset']
str_bytes = binary_data[str_start:]
# Deserialize
edges = deserialize_edges(edge_bytes, edge_count)
value_offsets = deserialize_value_array(val_bytes, entry_count, use_u32)
raw_table = decompress_string_table(str_bytes)
# Build adjacency from deserialized edges for lookup
# Reconstruct graph: node_id -> list of (stroke, target, skip)
adj = {}
node_finals = set()
edge_idx = 0
# We need to figure out which edges belong to which node.
# Edges are stored in DFS node order; is_last marks end of a node's edge list.
current_node = 0
node_edges = {}
i = 0
while i < len(edges):
stroke, target, skip, is_last = edges[i]
if current_node not in node_edges:
node_edges[current_node] = []
node_edges[current_node].append((stroke, target, skip))
if is_last:
current_node += 1
# Skip nodes that have no edges (they won't appear in edge list)
# We detect these by checking if next edge's parent should be higher
i += 1
# Now look up each entry via the reconstructed DAWG
correct = 0
wrong = 0
missing = 0
for strokes, expected in parsed_entries:
idx = dawg_lookup_index(root, strokes, skip_cache)
if idx < 0 or idx >= len(value_offsets):
missing += 1
continue
offset = value_offsets[idx]
# Find null terminator
try:
translation = lookup_string(raw_table, offset)
except (ValueError, IndexError):
missing += 1
continue
if translation == expected:
correct += 1
else:
wrong += 1
if wrong <= 5:
print(f" WRONG: strokes={strokes}, expected='{expected}', got='{translation}'")
return correct, wrong, missing
def print_stats(stats, target_kb=533):
"""Print compilation statistics."""
target_bytes = target_kb * 1024
total = stats['total_size']
pct = (total / target_bytes) * 100 if target_bytes else 0
print(f"Dictionary Compilation Stats:")
print(f" Entries: {stats['entry_count']:>10,}")
print(f" DAWG nodes: {stats['node_count']:>10,}")
print(f" DAWG edges: {stats['edge_count']:>10,}")
print(f" Bits/edge: {stats['bits_per_edge']:>10}")
print(f" ---")
print(f" Header: {stats['header_size']:>10,} bytes")
print(f" Edge array: {stats['edge_array_size']:>10,} bytes ({stats['edge_array_size']/1024:.1f} KB)")
print(f" Value array: {stats['value_array_size']:>10,} bytes ({stats['value_array_size']/1024:.1f} KB)")
print(f" String table: {stats['string_table_size']:>10,} bytes ({stats['string_table_size']/1024:.1f} KB)")
print(f" ---")
print(f" TOTAL: {total:>10,} bytes ({total/1024:.1f} KB)")
print(f" Budget: {target_bytes:>10,} bytes ({target_kb} KB)")
print(f" Usage: {pct:>9.1f}%")
if total <= target_bytes:
print(f" Status: FITS ({(target_bytes - total)/1024:.1f} KB remaining)")
else:
print(f" Status: OVER BUDGET by {(total - target_bytes)/1024:.1f} KB")
print(f" ---")
print(f" Split storage: {'yes' if stats['split_storage'] else 'no'}")
print(f" U32 offsets: {'yes' if stats['use_u32_offsets'] else 'no'}")
# ─── CLI ───
def main():
parser = argparse.ArgumentParser(
description='Compile steno dictionary to binary DAWG format')
parser.add_argument('input', nargs='?', default='/tmp/plover-main.json',
help='Path to JSON dictionary (default: /tmp/plover-main.json)')
parser.add_argument('--output', default='steno_dict.bin',
help='Output binary path (default: steno_dict.bin)')
parser.add_argument('--max-entries', type=int, default=130000,
help='Max entries to include (default: 130000)')
parser.add_argument('--target-size', type=int, default=533,
help='Target size in KB (default: 533)')
parser.add_argument('--split-storage', action='store_true',
help='Generate split-storage metadata header')
parser.add_argument('--stats', action='store_true',
help='Print detailed stats')
parser.add_argument('--verify', action='store_true',
help='Verify all entries after compilation')
args = parser.parse_args()
if not os.path.exists(args.input):
print(f"ERROR: Dictionary not found: {args.input}")
print("Download Plover dict:")
print(" curl -sL 'https://raw.githubusercontent.com/openstenoproject/plover/main/plover/assets/main.json' -o /tmp/plover-main.json")
sys.exit(1)
print(f"Compiling {args.input}...")
binary, stats, root, skip_cache, translations, parsed = \
compile_dictionary(args.input, args.max_entries, args.split_storage)
# Write output
with open(args.output, 'wb') as f:
f.write(binary)
print(f"Written {len(binary):,} bytes to {args.output}")
if args.stats:
print()
print_stats(stats, args.target_size)
if args.verify:
print()
print("Verifying...")
correct, wrong, missing = verify_compilation(binary, parsed, root, skip_cache)
total = correct + wrong + missing
print(f" Correct: {correct}/{total}")
print(f" Wrong: {wrong}/{total}")
print(f" Missing: {missing}/{total}")
if wrong > 0 or missing > 0:
print(" WARNING: Verification found errors!")
sys.exit(2)
else:
print(" All entries verified successfully.")
if __name__ == '__main__':
main()

106
tools/fetch_dict.py Normal file
View file

@ -0,0 +1,106 @@
#!/usr/bin/env python3
"""Download steno dictionary with caching.
Checks local file hash against known upstream hash.
If file exists and hash matches → skip download.
If file missing or hash mismatch → download fresh.
"""
import hashlib
import json
import os
import sys
import urllib.request
DICTS = {
"plover": {
"url": "https://raw.githubusercontent.com/openstenoproject/plover/main/plover/assets/main.json",
"filename": "plover-main.json",
},
"lapwing": {
"url": "https://raw.githubusercontent.com/aerickt/steno-dictionaries/main/lapwing-base.json",
"filename": "lapwing.json",
},
}
def sha256_file(path):
h = hashlib.sha256()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(65536), b''):
h.update(chunk)
return h.hexdigest()
def download(url, dest):
import subprocess
print(f"Downloading {url}...")
try:
urllib.request.urlretrieve(url, dest)
except urllib.error.URLError:
# Fallback to curl which uses system cert store
subprocess.check_call(["curl", "-sL", "-o", dest, url])
size = os.path.getsize(dest)
print(f"Downloaded {size:,} bytes to {dest}")
def fetch(dict_name, dest_dir):
if dict_name not in DICTS:
print(f"Unknown dict: {dict_name}. Available: {', '.join(DICTS.keys())}")
return 1
info = DICTS[dict_name]
dest = os.path.join(dest_dir, info["filename"])
hash_file = dest + ".sha256"
if os.path.exists(dest):
local_hash = sha256_file(dest)
if os.path.exists(hash_file):
with open(hash_file) as f:
cached_hash = f.read().strip()
if local_hash == cached_hash:
print(f"{dest} up to date (sha256={local_hash[:12]}...)")
return 0
try:
json.load(open(dest))
print(f"{dest} exists, valid JSON (sha256={local_hash[:12]}...)")
with open(hash_file, "w") as f:
f.write(local_hash)
return 0
except (json.JSONDecodeError, IOError):
print(f"{dest} corrupted, re-downloading")
os.makedirs(dest_dir, exist_ok=True)
download(info["url"], dest)
new_hash = sha256_file(dest)
with open(hash_file, "w") as f:
f.write(new_hash)
try:
with open(dest) as f:
d = json.load(f)
print(f"Verified: {len(d)} entries")
except (json.JSONDecodeError, IOError) as e:
print(f"WARNING: downloaded file invalid: {e}")
return 1
return 0
def main():
if len(sys.argv) < 2:
print(f"Usage: {sys.argv[0]} <plover|lapwing> [dest_dir]")
print(f" dest_dir defaults to ./dicts/")
sys.exit(1)
dict_name = sys.argv[1]
dest_dir = sys.argv[2] if len(sys.argv) > 2 else os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "dicts")
sys.exit(fetch(dict_name, dest_dir))
if __name__ == "__main__":
main()

464
tools/test_compiler.py Normal file
View file

@ -0,0 +1,464 @@
#!/usr/bin/env python3
"""Tests for dict_compiler.py — DAWG dictionary compiler."""
import json
import os
import struct
import sys
import tempfile
import pytest
# Ensure tools/ is importable
sys.path.insert(0, os.path.dirname(__file__))
from dict_compiler import (
STENO_KEYS,
FLAG_SPLIT_STORAGE,
HEADER_SIZE,
MAGIC,
VERSION,
build_dawg,
build_string_table,
compile_dictionary,
compute_skip_counts,
dawg_lookup_index,
decompress_string_table,
deserialize_edges,
deserialize_value_array,
get_dawg_traversal_order,
lookup_string,
parse_header,
parse_stroke,
parse_stroke_string,
serialize_edges,
serialize_header,
serialize_value_array,
trim_entries,
verify_compilation,
)
# ─── Small test dictionaries ───
SMALL_DICT = {
"S": "is",
"T": "it",
"K": "can",
"W": "with",
"H": "had",
"R": "are",
"TPHO": "no",
"STPH": "then",
"KAT": "cat",
"TKOG": "dog",
}
MULTI_STROKE_DICT = {
"S": "is",
"T": "it",
"KPA/HROL": "{}{-|}",
"TPHO/WUPB": "no one",
"K": "can",
}
MEDIUM_DICT = {
"S": "is",
"T": "it",
"K": "can",
"W": "with",
"H": "had",
"R": "are",
"A": "a",
"O": "oh",
"E": "he",
"U": "you",
"TPHO": "no",
"STPH": "then",
"KAT": "cat",
"TKOG": "dog",
"HOUS": "house",
"TPHAEUPL": "name",
"HROS": "also",
"TKPWRAET": "great",
"SKEL": "school",
"PLAS": "place",
}
def _make_temp_dict(d):
"""Write dict to temp JSON file, return path."""
fd, path = tempfile.mkstemp(suffix='.json')
with os.fdopen(fd, 'w') as f:
json.dump(d, f)
return path
def _parse_and_sort(d):
"""Parse dict entries and sort by stroke tuple."""
entries = []
for stroke_str, translation in d.items():
strokes = parse_stroke_string(stroke_str)
entries.append((strokes, translation))
entries.sort(key=lambda x: x[0])
return entries
# ─── Tests ───
class TestParseStroke:
"""Test stroke parsing."""
def test_left_side_stph(self):
"""STPH → S + T + P + H left side bits."""
val = parse_stroke("STPH")
expected = STENO_KEYS['S-'] | STENO_KEYS['T-'] | STENO_KEYS['P-'] | STENO_KEYS['H-']
assert val == expected
def test_right_side_eurb(self):
"""EURB → E + U + R + B right side bits."""
val = parse_stroke("EURB")
expected = STENO_KEYS['-E'] | STENO_KEYS['-U'] | STENO_KEYS['-R'] | STENO_KEYS['-B']
assert val == expected
def test_single_s(self):
"""S → just S bit."""
val = parse_stroke("S")
assert val == STENO_KEYS['S-']
def test_number_bar(self):
"""#STPH → number + S + T + P + H."""
val = parse_stroke("#STPH")
expected = (STENO_KEYS['#'] | STENO_KEYS['S-'] | STENO_KEYS['T-'] |
STENO_KEYS['P-'] | STENO_KEYS['H-'])
assert val == expected
def test_vowels(self):
"""AO → A + O vowel bits."""
val = parse_stroke("AO")
expected = STENO_KEYS['A-'] | STENO_KEYS['O-']
assert val == expected
def test_full_stroke(self):
"""STKPWHR → all left consonants."""
val = parse_stroke("STKPWHR")
expected = (STENO_KEYS['S-'] | STENO_KEYS['T-'] | STENO_KEYS['K-'] |
STENO_KEYS['P-'] | STENO_KEYS['W-'] | STENO_KEYS['H-'] |
STENO_KEYS['R-'])
assert val == expected
def test_star(self):
"""*E → star + E."""
val = parse_stroke("*E")
expected = STENO_KEYS['*'] | STENO_KEYS['-E']
assert val == expected
def test_multi_stroke_parse(self):
"""KPA/HROL parses to two stroke bitmasks."""
strokes = parse_stroke_string("KPA/HROL")
assert len(strokes) == 2
# First stroke: K + P + A
assert strokes[0] == (STENO_KEYS['K-'] | STENO_KEYS['P-'] | STENO_KEYS['A-'])
class TestBuildDawgSmall:
"""Test DAWG construction with small dictionary."""
def test_node_compression(self):
"""DAWG should have fewer nodes than a plain trie (compression happening)."""
entries = _parse_and_sort(SMALL_DICT)
root, node_count, edge_count = build_dawg(entries)
# With 10 entries, DAWG should have fewer nodes than
# total path length (which would be ~15+ for a trie)
assert node_count < 15
assert node_count > 0
assert edge_count > 0
def test_all_entries_reachable(self):
"""All entries should be reachable via traversal."""
entries = _parse_and_sort(SMALL_DICT)
root, node_count, edge_count = build_dawg(entries)
# Check each entry can be traversed
for strokes, _trans in entries:
node = root
for stroke in strokes:
assert stroke in node.edges, f"Missing edge for stroke in {strokes}"
node = node.edges[stroke]
assert node.final, f"Node not final for {strokes}"
class TestDawgLookup:
"""Test DAWG lookup via skip-count traversal."""
def test_all_lookups_correct(self):
"""All entries should have unique sequential skip-count indices."""
entries = _parse_and_sort(SMALL_DICT)
root, _, _ = build_dawg(entries)
skip_cache = compute_skip_counts(root)
indices = []
for strokes, _ in entries:
idx = dawg_lookup_index(root, strokes, skip_cache)
assert idx >= 0, f"Lookup failed for {strokes}"
indices.append(idx)
# All indices should be unique
assert len(set(indices)) == len(indices), "Duplicate indices found"
# Indices should be 0..n-1
assert sorted(indices) == list(range(len(entries)))
def test_missing_entry_returns_neg(self):
"""Looking up a non-existent stroke should return -1."""
entries = _parse_and_sort(SMALL_DICT)
root, _, _ = build_dawg(entries)
skip_cache = compute_skip_counts(root)
# A stroke not in the dict
fake_strokes = (0xDEAD,)
idx = dawg_lookup_index(root, fake_strokes, skip_cache)
assert idx == -1
def test_traversal_order_matches(self):
"""Traversal order should match sorted entries."""
entries = _parse_and_sort(SMALL_DICT)
root, _, _ = build_dawg(entries)
traversal = get_dawg_traversal_order(root)
entry_paths = [strokes for strokes, _ in entries]
assert traversal == entry_paths
class TestBinaryRoundTrip:
"""Test compile → binary → deserialize → verify."""
def test_small_dict_round_trip(self):
"""Compile small dict, deserialize, verify all lookups."""
path = _make_temp_dict(SMALL_DICT)
try:
binary, stats, root, skip_cache, translations, parsed = \
compile_dictionary(path, max_entries=100)
assert stats['entry_count'] == len(SMALL_DICT)
assert stats['total_size'] == len(binary)
assert stats['total_size'] > HEADER_SIZE
# Verify header
header = parse_header(binary)
assert header['magic'] == MAGIC
assert header['version'] == VERSION
assert header['entry_count'] == len(SMALL_DICT)
# Verify round-trip
correct, wrong, missing = verify_compilation(binary, parsed, root, skip_cache)
assert correct == len(SMALL_DICT)
assert wrong == 0
assert missing == 0
finally:
os.unlink(path)
def test_medium_dict_round_trip(self):
"""Medium dict round-trip."""
path = _make_temp_dict(MEDIUM_DICT)
try:
binary, stats, root, skip_cache, translations, parsed = \
compile_dictionary(path, max_entries=100)
correct, wrong, missing = verify_compilation(binary, parsed, root, skip_cache)
assert correct == len(MEDIUM_DICT)
assert wrong == 0
assert missing == 0
finally:
os.unlink(path)
def test_edge_serialization_round_trip(self):
"""Edge bit-packing round-trip."""
entries = _parse_and_sort(SMALL_DICT)
root, node_count, edge_count = build_dawg(entries)
edge_bytes, node_id_map, skip_cache = serialize_edges(root, node_count, edge_count)
edges = deserialize_edges(edge_bytes, edge_count)
assert len(edges) == edge_count
# Each edge should have valid fields
for stroke, target, skip, is_last in edges:
assert 0 <= stroke <= 0xFFFF
assert 0 <= target <= 0xFFFF
assert 0 <= skip <= 0x1FFFF
def test_string_table_round_trip(self):
"""String table compress/decompress round-trip."""
translations = list(SMALL_DICT.values())
table_bytes, offsets, raw_size = build_string_table(translations)
raw = decompress_string_table(table_bytes)
for i, trans in enumerate(translations):
recovered = lookup_string(raw, offsets[i])
assert recovered == trans, f"Mismatch at {i}: '{trans}' vs '{recovered}'"
class TestEntryTrimming:
"""Test entry trimming logic."""
def test_trim_keeps_single_stroke(self):
"""With max_entries < total, single-stroke entries kept preferentially."""
d = {}
# 10 unique single-stroke entries (no duplicates)
single_strokes = ["S", "T", "K", "W", "H", "A", "O", "-F", "-B", "-G"]
for i, k in enumerate(single_strokes):
d[k] = f"word_{i}"
# 40 unique multi-stroke entries using distinct second strokes
left = ["S", "T", "K", "P", "W", "H", "R", "SK", "TK", "PH"]
right = ["-F", "-R", "-P", "-B"]
idx = 0
for lk in left:
for rk in right:
d[f"KAT/{lk}{rk}"] = f"long_translation_{idx}"
idx += 1
# Total = 50 entries
entries = list(d.items())
assert len(entries) == 50, f"Expected 50, got {len(entries)}"
trimmed = trim_entries(entries, 30)
assert len(trimmed) == 30
# Count single vs multi in result
single_count = sum(1 for s, _ in trimmed if '/' not in s)
multi_count = sum(1 for s, _ in trimmed if '/' in s)
# All 10 single-stroke entries should be kept
assert single_count == 10
assert multi_count == 20
def test_no_trim_when_under_limit(self):
"""No trimming when entries < max_entries."""
entries = list(SMALL_DICT.items())
trimmed = trim_entries(entries, 1000)
assert len(trimmed) == len(entries)
def test_trim_multi_stroke_by_length(self):
"""Multi-stroke entries trimmed by translation length (shorter kept)."""
d = {"S": "is"} # 1 single-stroke
# Add multi-stroke with varying translation lengths
d["KAT/S"] = "ab" # short
d["KAT/T"] = "abcdefghij" # long
d["KAT/K"] = "abc" # medium
entries = list(d.items())
trimmed = trim_entries(entries, 3)
# Should keep: single("S"), then shortest multi-stroke
assert len(trimmed) == 3
trans = [t for _, t in trimmed]
assert "is" in trans # single stroke kept
assert "ab" in trans # shortest multi kept
assert "abc" in trans # medium kept
assert "abcdefghij" not in trans # longest dropped
class TestSplitStorageFlag:
"""Test --split-storage flag."""
def test_flag_set_in_header(self):
"""split_storage flag should set bit 0 in header flags."""
path = _make_temp_dict(SMALL_DICT)
try:
binary, stats, _, _, _, _ = compile_dictionary(
path, max_entries=100, split_storage=True)
header = parse_header(binary)
assert header['flags'] & FLAG_SPLIT_STORAGE != 0
assert stats['split_storage'] is True
finally:
os.unlink(path)
def test_flag_not_set_by_default(self):
"""split_storage flag should NOT be set by default."""
path = _make_temp_dict(SMALL_DICT)
try:
binary, stats, _, _, _, _ = compile_dictionary(
path, max_entries=100, split_storage=False)
header = parse_header(binary)
assert header['flags'] & FLAG_SPLIT_STORAGE == 0
assert stats['split_storage'] is False
finally:
os.unlink(path)
class TestMultiStroke:
"""Test multi-stroke entry handling."""
def test_multi_stroke_compile_and_lookup(self):
"""Multi-stroke entries (e.g. KPA/HROL) should compile and look up correctly."""
path = _make_temp_dict(MULTI_STROKE_DICT)
try:
binary, stats, root, skip_cache, translations, parsed = \
compile_dictionary(path, max_entries=100)
correct, wrong, missing = verify_compilation(binary, parsed, root, skip_cache)
assert correct == len(MULTI_STROKE_DICT)
assert wrong == 0
assert missing == 0
finally:
os.unlink(path)
def test_multi_stroke_traversal(self):
"""Multi-stroke entries should appear in correct traversal order."""
entries = _parse_and_sort(MULTI_STROKE_DICT)
root, _, _ = build_dawg(entries)
skip_cache = compute_skip_counts(root)
for strokes, _ in entries:
idx = dawg_lookup_index(root, strokes, skip_cache)
assert idx >= 0, f"Multi-stroke lookup failed: {strokes}"
def test_kpa_hrol_specific(self):
"""KPA/HROL → {}{-|} specifically."""
d = {"KPA/HROL": "{}{-|}"}
entries = _parse_and_sort(d)
root, _, _ = build_dawg(entries)
skip_cache = compute_skip_counts(root)
strokes = parse_stroke_string("KPA/HROL")
idx = dawg_lookup_index(root, strokes, skip_cache)
assert idx == 0 # only entry → index 0
class TestHeaderSerialization:
"""Test header pack/unpack."""
def test_header_size(self):
"""Header should be exactly 32 bytes."""
header = serialize_header(0, 100, 50, 200, 1000, 500, 800)
assert len(header) == HEADER_SIZE
def test_header_round_trip(self):
"""Header fields should survive pack/unpack."""
header = serialize_header(
flags=FLAG_SPLIT_STORAGE,
entry_count=12345,
node_count=6789,
edge_count=11111,
string_table_offset=99999,
string_table_size=55555,
value_array_offset=44444,
)
parsed = parse_header(header)
assert parsed['magic'] == MAGIC
assert parsed['version'] == VERSION
assert parsed['flags'] == FLAG_SPLIT_STORAGE
assert parsed['entry_count'] == 12345
assert parsed['node_count'] == 6789
assert parsed['edge_count'] == 11111
assert parsed['string_table_offset'] == 99999
assert parsed['string_table_size'] == 55555
assert parsed['value_array_offset'] == 44444
if __name__ == '__main__':
pytest.main([__file__, '-v'])

3
zephyr/module.yml Normal file
View file

@ -0,0 +1,3 @@
build:
cmake: .
kconfig: Kconfig