Scaffolding: ZMK module structure, Kconfig, DTS, behavior driver

- zephyr/module.yml with dts_root for dt-bindings
- Kconfig: STENO_ENGINE, dict selection (Plover/Lapwing/test),
  MPHF toggle, Unicode modes, history size, multi-stroke timeout
- DTS behavior binding (one_param, steno key index)
- behavior_steno.c: chord accumulation, all-up detection,
  multi-stroke buffering with timeout, star undo, formatter pipeline,
  3-way dict dispatch (split/MPHF/simple trie)
- dict_embed.S: .incbin from generated header path
- steno_keys.h: 23-key layout, bit positions matching compiler
This commit is contained in:
afiqzudinhadi 2026-07-02 02:04:28 +08:00
parent 234bc8d731
commit 1ec204b845
33 changed files with 5443 additions and 0 deletions

6
.gitignore vendored
View file

@ -0,0 +1,6 @@
__pycache__/
*.pyc
dicts/plover-main.json
dicts/lapwing.json
dicts/*.sha256
tests/build/

105
CMakeLists.txt Normal file
View file

@ -0,0 +1,105 @@
# Copyright (c) 2024 Afiq Zudin Hadi
# SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
if(CONFIG_STENO_ENGINE)
if(NOT CONFIG_ZMK_SPLIT OR CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
target_sources(app PRIVATE
src/behavior_steno.c
src/output.c
src/formatter.c
src/undo.c
src/dict_embed.S
)
if(CONFIG_STENO_DICT_MPHF)
target_sources(app PRIVATE src/dict_mphf.c)
else()
target_sources(app PRIVATE src/trie.c)
endif()
target_include_directories(app PRIVATE
include
${CMAKE_CURRENT_SOURCE_DIR}/src
)
# ── Dictionary source resolution ──
find_package(Python3 REQUIRED COMPONENTS Interpreter)
set(STENO_DICT_BIN ${CMAKE_CURRENT_BINARY_DIR}/steno_dict.bin)
set(STENO_DICTS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/dicts)
set(STENO_FETCH ${CMAKE_CURRENT_SOURCE_DIR}/tools/fetch_dict.py)
if(CONFIG_STENO_DICT_PLOVER)
set(STENO_DICT_NAME "plover")
set(STENO_DICT_SRC ${STENO_DICTS_DIR}/plover-main.json)
elseif(CONFIG_STENO_DICT_LAPWING)
set(STENO_DICT_NAME "lapwing")
set(STENO_DICT_SRC ${STENO_DICTS_DIR}/lapwing.json)
else()
set(STENO_DICT_SRC ${STENO_DICTS_DIR}/test.json)
endif()
# Auto-download dict if needed (Plover/Lapwing only)
if(DEFINED STENO_DICT_NAME AND NOT EXISTS ${STENO_DICT_SRC})
message(STATUS "Steno: downloading ${STENO_DICT_NAME} dictionary...")
execute_process(
COMMAND ${Python3_EXECUTABLE} ${STENO_FETCH}
${STENO_DICT_NAME} ${STENO_DICTS_DIR}
RESULT_VARIABLE FETCH_RESULT
)
if(NOT FETCH_RESULT EQUAL 0)
message(WARNING "Steno: dict download failed. Build may fail.")
endif()
endif()
# ── Dictionary compilation ──
if(EXISTS ${STENO_DICT_SRC})
if(CONFIG_STENO_DICT_MPHF)
# Fetch at build time if hash changed (re-run on rebuild)
if(DEFINED STENO_DICT_NAME)
add_custom_command(
OUTPUT ${STENO_DICT_SRC}.stamp
COMMAND ${Python3_EXECUTABLE} ${STENO_FETCH}
${STENO_DICT_NAME} ${STENO_DICTS_DIR}
COMMAND ${CMAKE_COMMAND} -E touch ${STENO_DICT_SRC}.stamp
COMMENT "Checking ${STENO_DICT_NAME} dictionary for updates"
)
add_custom_target(steno_dict_fetch DEPENDS ${STENO_DICT_SRC}.stamp)
endif()
add_custom_command(
OUTPUT ${STENO_DICT_BIN}
COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_mphf.py
${STENO_DICT_SRC} ${STENO_DICT_BIN}
--max-size ${CONFIG_STENO_DICT_MAX_SIZE}
DEPENDS ${STENO_DICT_SRC}
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_mphf.py
COMMENT "Compiling steno dictionary (MPHF)"
)
else()
add_custom_command(
OUTPUT ${STENO_DICT_BIN}
COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_simple.py
${STENO_DICT_SRC} -o ${STENO_DICT_BIN}
DEPENDS ${STENO_DICT_SRC}
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_simple.py
COMMENT "Compiling steno dictionary (simple)"
)
endif()
add_custom_target(steno_dict_gen DEPENDS ${STENO_DICT_BIN})
if(TARGET steno_dict_fetch)
add_dependencies(steno_dict_gen steno_dict_fetch)
endif()
add_dependencies(app steno_dict_gen)
# Generate header with dict path for .incbin
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/steno_dict_path.h
"#define STENO_DICT_BIN_PATH \"${STENO_DICT_BIN}\"\n")
target_include_directories(app PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
endif()
endif() # central role
endif() # CONFIG_STENO_ENGINE

85
Kconfig Normal file
View file

@ -0,0 +1,85 @@
# Copyright (c) 2024 Afiq Zudin Hadi
# SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
menuconfig STENO_ENGINE
bool "Steno Engine"
default n
help
Enable the stenography engine for ZMK.
if STENO_ENGINE
choice STENO_DICT
prompt "Steno dictionary"
default STENO_DICT_PLOVER
config STENO_DICT_PLOVER
bool "Plover main dictionary (MPHF compressed)"
help
Use Plover main.json via MPHF compression (~44K entries in 453KB).
config STENO_DICT_LAPWING
bool "Lapwing dictionary (MPHF compressed)"
config STENO_DICT_TEST
bool "Test dictionary (46 entries, simple format)"
help
Small built-in test dictionary for development.
endchoice
config STENO_DICT_MPHF
bool
default y if STENO_DICT_PLOVER || STENO_DICT_LAPWING
select ZLIB
help
Use MPHF (minimal perfect hash) dictionary format.
Selects ZLIB for block-compressed string table decompression.
Auto-selected for Plover/Lapwing dicts.
config STENO_CUSTOM_KEYMAP
bool "Custom steno keymap"
default n
choice STENO_UNICODE_MODE
prompt "Unicode output mode"
default STENO_UNICODE_MODE_NONE
config STENO_UNICODE_MODE_NONE
bool "None (HID only)"
config STENO_UNICODE_MODE_LINUX
bool "Linux (IBus)"
config STENO_UNICODE_MODE_MACOS
bool "macOS"
config STENO_UNICODE_MODE_WINC
bool "Windows (WinCompose)"
endchoice
config STENO_HISTORY_SIZE
int "Stroke history size (undo depth)"
default 100
range 10 500
config STENO_KEY_DELAY_MS
int "Key output delay (ms)"
default 0
range 0 50
config STENO_MULTI_STROKE_TIMEOUT_MS
int "Multi-stroke timeout (ms)"
default 500
range 100 5000
config STENO_DICT_MAX_SIZE
int "Max dictionary binary size (bytes)"
default 430080
help
Max compiled dict size. 430080 = 420KB.
Leaves room for zlib decompressor + USB logging overhead.
The MPHF compiler auto-trims to fit.
endif # STENO_ENGINE

132
LICENSE Normal file
View file

@ -0,0 +1,132 @@
# PolyForm Noncommercial License 1.0.0
<https://polyformproject.org/licenses/noncommercial/1.0.0>
## Acceptance
In order to get any license under these terms, you must agree
to them as both strict obligations and conditions to all
your licenses.
## Copyright License
The licensor grants you a copyright license for the software
to do everything you might do with the software that would
otherwise infringe the licensor's copyright in it for any
permitted purpose. However, you may only distribute the
software according to [Distribution License](#distribution-license)
and make changes or new works based on the software according
to [Changes and New Works License](#changes-and-new-works-license).
## Distribution License
The licensor grants you an additional copyright license to
distribute copies of the software. Your license to distribute
covers distributing the software with changes and new works
permitted by [Changes and New Works License](#changes-and-new-works-license).
## Notices
You must ensure that anyone who gets a copy of any part of
the software from you also gets a copy of these terms or the
URL for them above, as well as copies of any plain-text lines
beginning with `Required Notice:` that the licensor provided
with the software. For example:
> Required Notice: Copyright Afiq Zudin Hadi (https://github.com/afiqzudinhadi)
## Changes and New Works License
The licensor grants you an additional copyright license to
make changes and new works based on the software for any
permitted purpose.
## Patent License
The licensor grants you a patent license for the software that
covers patent claims the licensor can license, or becomes able
to license, that you would infringe by using the software.
## Noncommercial Purposes
Any noncommercial purpose is a permitted purpose.
## Personal Uses
Personal use for research, experiment, and testing for
the benefit of public knowledge, personal study, private
entertainment, hobby projects, amateur pursuits, or religious
observance, without any anticipated commercial application,
is use for a permitted purpose.
## Noncommercial Organizations
Use by any charitable organization, educational institution,
public research organization, public safety or health
organization, environmental protection organization, or
government institution is use for a permitted purpose
regardless of the source of funding or obligations resulting
from the funding.
## Fair Use
You may have "fair use" rights for the software under the
law. These terms do not limit them.
## No Other Rights
These terms do not allow you to sublicense or transfer any
of your licenses to anyone else, or prevent the licensor
from granting licenses to anyone else. These terms do not
imply any other licenses.
## Patent Defense
If you make any written claim that the software infringes
or contributes to infringement of any patent, your patent
license for the software granted under these terms ends
immediately. If your company makes such a claim, your
patent license ends immediately for work on behalf of your
company.
## Violations
The first time you are notified in writing that you have
violated any of these terms, or done anything with the
software not covered by your licenses, your licenses can
nonetheless continue if you come into full compliance with
these terms, and take practical steps to correct past
violations, within 32 days of receiving notice. Otherwise,
all your licenses end immediately.
## No Liability
As far as the law allows, the software comes as is, without
any warranty or condition, and the licensor will not be
liable to you for any damages arising out of these terms or
the use or nature of the software, under any kind of legal
claim.
## Definitions
The **licensor** is the individual or entity offering these
terms, and the **software** is the software the licensor
makes available under these terms.
**You** refers to the individual or entity agreeing to these
terms.
**Your company** is any legal entity, sole proprietorship,
or other kind of organization that you work for, plus all
organizations that have control over, are under the control
of, or are under common control with that organization.
**Control** means ownership of substantially all the assets
of an entity, or the power to direct its management and
policies by vote, contract, or otherwise. Control can be
direct or indirect.
**Your licenses** are all the licenses granted to you for
the software under these terms.
**Use** means anything you do with the software requiring
one of your licenses.

66
README.md Normal file
View file

@ -0,0 +1,66 @@
# zmk-steno-engine
Clean-room stenography engine for [ZMK Firmware](https://zmk.dev). Dictionary-based lookup with multi-stroke support, optimized for nRF52840 flash constraints.
**Status:** Early development — basic single/multi-stroke lookup works, formatter not yet implemented.
## Features
- Standard 23-key steno layout
- Sorted-array dictionary with binary search lookup
- Multi-stroke support with configurable timeout
- All-up chord detection
- HID keyboard output (ASCII)
- Build-time dictionary compilation from Plover JSON
- Test dictionary included for development
## Building
Add as a ZMK module in your `west.yml`:
```yaml
manifest:
remotes:
- name: zmk-steno-engine
url-base: https://github.com/afiqzudinhadi
projects:
- name: zmk-steno-engine
remote: zmk-steno-engine
revision: optimize-dict
```
Enable in your `.conf`:
```
CONFIG_STENO_ENGINE=y
```
Use in your keymap:
```dts
#include <dt-bindings/zmk/steno_keys.h>
/ {
keymap {
steno_layer {
bindings = <
&steno STENO_SL &steno STENO_TL &steno STENO_PL ...
>;
};
};
};
```
## Dictionary Compiler
```bash
# Compile test dictionary
python3 tools/compile_simple.py dicts/test.json -o steno_dict.bin --stats
# Compile Plover dictionary (trimmed)
python3 tools/compile_simple.py plover-main.json -o steno_dict.bin --max-entries 120000 --stats
```
## License
[PolyForm Noncommercial 1.0.0](LICENSE)

48
dicts/test.json Normal file
View file

@ -0,0 +1,48 @@
{
"S": "is",
"T": "it",
"-T": "the",
"K": "can",
"W": "with",
"H": "had",
"R": "are",
"A": "a",
"AOEU": "I",
"TKPW": "go",
"HRAO": "hello",
"WORBGD": "world",
"KAT": "cat",
"TKOG": "dog",
"HOUS": "house",
"TPEUR": "fire",
"WAURT": "water",
"TPOOD": "food",
"PWO*BG": "book",
"KO": "co",
"KO/PHAOURD": "computer",
"KO/PHAOURDZ": "computers",
"TEFT": "test",
"TEFTD": "tested",
"TEFTG": "testing",
"STE/TPHO": "steno",
"STE/TPHOG": "stenography",
"KPA": "{}{-|}",
"S-P": "{^ ^}",
"TP-PL": "{.}",
"KW-BG": "{,}",
"H-F": "{?}",
"SKHRAPL": "{!}",
"*": "{*}",
"TPHO": "no",
"KWRA": "yes",
"THA": "that",
"PWUT": "but",
"TPHOT": "not",
"SREU": "have",
"WHA": "what",
"HEU": "he",
"SHEU": "she",
"THE": "they",
"WE": "we",
"KWREU": "I"
}

View file

@ -0,0 +1,13 @@
/*
* Copyright (c) 2024 Afiq Zudin Hadi
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*/
/ {
behaviors {
/omit-if-no-ref/ steno: steno_engine {
compatible = "zmk,behavior-steno-engine";
#binding-cells = <1>;
};
};
};

View file

@ -0,0 +1,14 @@
# Copyright (c) 2024 Afiq Zudin Hadi
# SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
description: |
Steno engine behavior. Maps physical keys to steno positions,
detects chords on all-keys-up, and emits translated text.
Usage in keymap:
&steno STENO_SL // left S key
&steno STENO_TL // left T key
compatible: "zmk,behavior-steno-engine"
include: one_param.yaml

View file

@ -0,0 +1,53 @@
/*
* Copyright (c) 2024 Afiq Zudin Hadi
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*
* Standard steno key layout. Each key = bit position in uint32_t chord.
* Bit assignments match the compiler's STENO_KEYS bitmask exactly.
*
* Layout (standard steno order):
* S T K P W H R A O * E U F R P B L G T S D Z #
* 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
*/
#ifndef DT_BINDINGS_ZMK_STENO_KEYS_H_
#define DT_BINDINGS_ZMK_STENO_KEYS_H_
/* Left hand consonants */
#define STENO_SL 0 /* S- (left) */
#define STENO_TL 1 /* T- */
#define STENO_KL 2 /* K- */
#define STENO_PL 3 /* P- */
#define STENO_WL 4 /* W- */
#define STENO_HL 5 /* H- */
#define STENO_RL 6 /* R- (left) */
/* Vowels */
#define STENO_A 7 /* A */
#define STENO_O 8 /* O */
/* Center */
#define STENO_STAR 9 /* * */
/* Vowels (right) */
#define STENO_E 10 /* E */
#define STENO_U 11 /* U */
/* Right hand consonants */
#define STENO_FR 12 /* -F */
#define STENO_RR 13 /* -R (right) */
#define STENO_PR 14 /* -P (right) */
#define STENO_BR 15 /* -B */
#define STENO_LR 16 /* -L */
#define STENO_GR 17 /* -G */
#define STENO_TR 18 /* -T (right) */
#define STENO_SR 19 /* -S (right) */
#define STENO_DR 20 /* -D */
#define STENO_ZR 21 /* -Z */
/* Number bar */
#define STENO_NUM 22 /* # */
#define STENO_KEY_COUNT 23
#endif /* DT_BINDINGS_ZMK_STENO_KEYS_H_ */

271
src/behavior_steno.c Normal file
View file

@ -0,0 +1,271 @@
/*
* Copyright (c) 2024 Afiq Zudin Hadi
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*/
#define DT_DRV_COMPAT zmk_behavior_steno_engine
#include <zephyr/device.h>
#include <zephyr/kernel.h>
#include <zephyr/logging/log.h>
#include <drivers/behavior.h>
#include <zmk/behavior.h>
#include "output.h"
#include "undo.h"
#include "formatter.h"
#if IS_ENABLED(CONFIG_STENO_DICT_MPHF)
#include "dict_mphf.h"
#else
#include "trie.h"
#endif
LOG_MODULE_DECLARE(zmk, CONFIG_ZMK_LOG_LEVEL);
extern const uint8_t _steno_dict_start[];
extern const uint8_t _steno_dict_end[];
#if IS_ENABLED(CONFIG_STENO_DICT_MPHF)
static struct dict_mphf mphf_dict;
#endif
#define STENO_MAX_MULTI 8
#define STENO_MULTI_TIMEOUT_MS CONFIG_STENO_MULTI_STROKE_TIMEOUT_MS
static inline const char *dict_lookup(const uint32_t *strokes, uint8_t count)
{
#if IS_ENABLED(CONFIG_STENO_DICT_MPHF)
return dict_mphf_lookup(&mphf_dict, strokes, count);
#else
return steno_trie_lookup(strokes, count);
#endif
}
static inline bool dict_has_prefix(const uint32_t *strokes, uint8_t count)
{
#if IS_ENABLED(CONFIG_STENO_DICT_MPHF)
return (count == 1) ? dict_mphf_has_prefix(&mphf_dict, strokes[0]) : false;
#else
return steno_trie_has_prefix(strokes, count);
#endif
}
struct steno_state {
uint32_t current_chord;
uint8_t keys_held;
uint32_t pending_strokes[STENO_MAX_MULTI];
uint8_t stroke_count;
struct k_work_delayable multi_timeout;
};
static struct steno_state state;
static struct steno_fmt_state fmt_state;
static struct stroke_history undo_history;
static bool dict_ready;
static void flush_strokes(void);
static void multi_timeout_handler(struct k_work *work);
static void emit_formatted(const char *translation,
const uint32_t *strokes, uint8_t stroke_count)
{
struct steno_fmt_result result;
steno_fmt_process(&fmt_state, translation, &result);
if (result.backspaces > 0) {
steno_output_backspace(result.backspaces);
}
if (result.len > 0) {
steno_output_send(result.text, result.len);
}
if (!result.is_command_only) {
steno_undo_push(&undo_history, strokes, stroke_count,
result.len + result.backspaces, 0, 0);
}
}
static void process_chord(void)
{
if (state.current_chord == 0) {
return;
}
/* Star-only stroke (bit 9) → undo */
if (state.current_chord == (1U << 9)) {
state.current_chord = 0;
struct stroke_history_entry *ue = steno_undo_pop(&undo_history);
if (ue) {
steno_output_backspace(ue->output_len);
}
return;
}
k_work_cancel_delayable(&state.multi_timeout);
if (state.stroke_count < STENO_MAX_MULTI) {
state.pending_strokes[state.stroke_count] = state.current_chord;
state.stroke_count++;
} else {
flush_strokes();
state.pending_strokes[0] = state.current_chord;
state.stroke_count = 1;
}
state.current_chord = 0;
if (!dict_ready) {
LOG_WRN("steno dict not ready, flushing");
flush_strokes();
return;
}
const char *translation = dict_lookup(
state.pending_strokes, state.stroke_count);
LOG_INF("steno lookup %u strokes → %s", state.stroke_count,
translation ? translation : "(null)");
if (translation) {
emit_formatted(translation, state.pending_strokes, state.stroke_count);
state.stroke_count = 0;
return;
}
if (dict_has_prefix(state.pending_strokes, state.stroke_count)) {
k_work_schedule(&state.multi_timeout,
K_MSEC(STENO_MULTI_TIMEOUT_MS));
return;
}
if (state.stroke_count > 1) {
uint32_t last = state.pending_strokes[state.stroke_count - 1];
state.stroke_count--;
const char *partial = dict_lookup(
state.pending_strokes, state.stroke_count);
if (partial) {
emit_formatted(partial, state.pending_strokes, state.stroke_count);
}
state.pending_strokes[0] = last;
state.stroke_count = 1;
const char *rest = dict_lookup(&last, 1);
if (rest) {
emit_formatted(rest, &last, 1);
state.stroke_count = 0;
}
return;
}
flush_strokes();
}
static void flush_strokes(void)
{
LOG_DBG("Flushing %u untranslated strokes", state.stroke_count);
state.stroke_count = 0;
}
static void multi_timeout_handler(struct k_work *work)
{
ARG_UNUSED(work);
if (state.stroke_count == 0) {
return;
}
const char *translation = dict_lookup(
state.pending_strokes, state.stroke_count);
if (translation) {
emit_formatted(translation, state.pending_strokes, state.stroke_count);
}
state.stroke_count = 0;
}
static int on_steno_binding_pressed(struct zmk_behavior_binding *binding,
struct zmk_behavior_binding_event event)
{
uint32_t key_index = binding->param1;
if (key_index > 35) {
return -EINVAL;
}
state.current_chord |= (1U << key_index);
state.keys_held++;
LOG_INF("steno press key=%u chord=0x%06X held=%u",
key_index, state.current_chord, state.keys_held);
return ZMK_BEHAVIOR_OPAQUE;
}
static int on_steno_binding_released(struct zmk_behavior_binding *binding,
struct zmk_behavior_binding_event event)
{
if (state.keys_held > 0) {
state.keys_held--;
}
LOG_INF("steno release held=%u chord=0x%06X", state.keys_held, state.current_chord);
if (state.keys_held == 0 && state.current_chord != 0) {
LOG_INF("steno all-up → process chord 0x%06X", state.current_chord);
process_chord();
}
return ZMK_BEHAVIOR_OPAQUE;
}
static int behavior_steno_init(const struct device *dev)
{
ARG_UNUSED(dev);
state.current_chord = 0;
state.keys_held = 0;
state.stroke_count = 0;
steno_fmt_init(&fmt_state);
steno_undo_init(&undo_history);
k_work_init_delayable(&state.multi_timeout, multi_timeout_handler);
size_t dict_size = _steno_dict_end - _steno_dict_start;
if (dict_size > 4) {
int ret;
#if IS_ENABLED(CONFIG_STENO_DICT_MPHF)
ret = dict_mphf_init(&mphf_dict, _steno_dict_start, dict_size);
#else
ret = steno_trie_init(_steno_dict_start, dict_size);
#endif
if (ret == 0) {
dict_ready = true;
LOG_INF("Steno dict loaded (%u bytes)", (unsigned)dict_size);
} else {
LOG_ERR("Steno dict init failed: %d", ret);
}
} else {
LOG_WRN("No steno dict embedded");
}
LOG_INF("Steno engine initialized");
return 0;
}
static const struct behavior_driver_api steno_driver_api = {
.locality = BEHAVIOR_LOCALITY_CENTRAL,
.binding_pressed = on_steno_binding_pressed,
.binding_released = on_steno_binding_released,
};
#define STENO_INST(n) \
BEHAVIOR_DT_INST_DEFINE(n, \
behavior_steno_init, \
NULL, \
NULL, \
NULL, \
POST_KERNEL, \
CONFIG_KERNEL_INIT_PRIORITY_DEFAULT, \
&steno_driver_api);
DT_INST_FOREACH_STATUS_OKAY(STENO_INST)

14
src/dict_embed.S Normal file
View file

@ -0,0 +1,14 @@
#include "steno_dict_path.h"
.section .rodata.steno_dict, "a", %progbits
.global _steno_dict_start
.global _steno_dict_end
.balign 4
_steno_dict_start:
#ifdef STENO_DICT_BIN_PATH
.incbin STENO_DICT_BIN_PATH
#else
.byte 0x00, 0x00, 0x00, 0x00
#endif
_steno_dict_end:

323
src/dict_mphf.c Normal file
View file

@ -0,0 +1,323 @@
/**
* MPHF dictionary lookup engine implementation.
*
* All data is read directly from flash. No heap allocation.
* String table is block-compressed with zlib; decompressed on-demand
* into a static 4KB buffer.
*
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*/
#include "dict_mphf.h"
#include <string.h>
#ifdef __ZEPHYR__
#include <zephyr/sys/crc.h>
#endif
/* ─── Minimal inflate for non-Zephyr (native tests) ─── */
#ifndef __ZEPHYR__
#include <stdlib.h>
/* Use zlib on host for native tests */
#ifdef HAS_ZLIB
#include <zlib.h>
static int block_inflate(const uint8_t *src, size_t src_len,
uint8_t *dst, size_t dst_cap, size_t *dst_len)
{
uLongf out_len = dst_cap;
int ret = uncompress(dst, &out_len, src, src_len);
if (ret == Z_OK) {
*dst_len = out_len;
return 0;
}
return -1;
}
#else
/* Minimal tinf inflate — bundled for host-only testing.
* On Zephyr, we use the kernel's built-in zlib. */
#include "tinf/tinf.h"
static int block_inflate(const uint8_t *src, size_t src_len,
uint8_t *dst, size_t dst_cap, size_t *dst_len)
{
unsigned int out_len = dst_cap;
/* Skip 2-byte zlib header, strip 4-byte adler32 checksum */
if (src_len < 6) return -1;
int ret = tinf_uncompress(dst, &out_len, src + 2, src_len - 6);
if (ret == 0) {
*dst_len = out_len;
return 0;
}
return -1;
}
#endif /* HAS_ZLIB */
#else /* __ZEPHYR__ */
#include <zephyr/sys/util.h>
/* Zephyr built-in zlib decompression */
#if __has_include(<zephyr/lib/zlib/zlib.h>)
#include <zephyr/lib/zlib/zlib.h>
#elif __has_include(<zlib.h>)
#include <zlib.h>
#endif
static int block_inflate(const uint8_t *src, size_t src_len,
uint8_t *dst, size_t dst_cap, size_t *dst_len)
{
/* Try Zephyr's tinycrypt/miniz or fall back to raw copy */
#if defined(CONFIG_ZLIB)
uLongf out_len = dst_cap;
int ret = uncompress(dst, &out_len, src, src_len);
if (ret == Z_OK) {
*dst_len = out_len;
return 0;
}
return -1;
#else
/* No zlib available — strings must be uncompressed (flags bit 0 clear) */
(void)src; (void)src_len; (void)dst; (void)dst_cap; (void)dst_len;
return -1;
#endif
}
#endif /* __ZEPHYR__ */
/* ─── FNV-1a 32-bit hash ─── */
static uint32_t fnv1a_32(const uint8_t *data, size_t len)
{
uint32_t h = 0x811c9dc5u;
for (size_t i = 0; i < len; i++) {
h ^= data[i];
h *= 0x01000193u;
}
return h;
}
static uint32_t hash_key(const uint8_t *key, size_t key_len, uint32_t seed)
{
uint32_t h = 0x811c9dc5u;
uint8_t seed_bytes[4];
seed_bytes[0] = (uint8_t)(seed);
seed_bytes[1] = (uint8_t)(seed >> 8);
seed_bytes[2] = (uint8_t)(seed >> 16);
seed_bytes[3] = (uint8_t)(seed >> 24);
for (int i = 0; i < 4; i++) {
h ^= seed_bytes[i];
h *= 0x01000193u;
}
for (size_t i = 0; i < key_len; i++) {
h ^= key[i];
h *= 0x01000193u;
}
return h;
}
/* ─── Bit-packed field reading ─── */
static uint32_t read_bits(const uint8_t *data, uint32_t bit_pos, uint8_t n_bits)
{
uint32_t value = 0;
for (uint8_t i = 0; i < n_bits; i++) {
uint32_t byte_idx = (bit_pos + i) / 8;
uint8_t bit_idx = (bit_pos + i) % 8;
if (data[byte_idx] & (1u << bit_idx)) {
value |= (1u << i);
}
}
return value;
}
static inline uint32_t align4(uint32_t n)
{
return (n + 3u) & ~3u;
}
/* ─── Init ─── */
int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len)
{
if (!dict || !data) return -1;
if (len < sizeof(struct dict_mphf_header)) return -2;
const struct dict_mphf_header *hdr = (const struct dict_mphf_header *)data;
if (hdr->magic != DICT_MPHF_MAGIC) return -3;
if (hdr->version != DICT_MPHF_VERSION) return -4;
dict->header = hdr;
const uint8_t *base = (const uint8_t *)data;
uint32_t offset = sizeof(struct dict_mphf_header);
/* Displacements */
dict->displacements = base + offset;
uint32_t disp_bits_total = (uint32_t)hdr->bucket_count * hdr->disp_bits;
dict->disp_section_len = align4((disp_bits_total + 7) / 8);
offset += dict->disp_section_len;
/* Values */
dict->values = base + offset;
uint32_t val_bits_total = (uint32_t)hdr->entry_count * hdr->value_bits;
dict->val_section_len = align4((val_bits_total + 7) / 8);
offset += dict->val_section_len;
/* Fingerprints */
dict->fingerprints = base + offset;
dict->fp_section_len = align4(hdr->entry_count);
offset += dict->fp_section_len;
/* String offsets (u24 LE, 3 bytes each) */
dict->string_offsets = base + offset;
offset += hdr->unique_count * 3;
/* String data section */
dict->str_data_start = base + offset;
if (hdr->flags & DICT_MPHF_FLAG_COMPRESSED) {
/* Block directory: u16 block_count + u32[] offsets */
dict->block_count = dict->str_data_start[0] |
((uint16_t)dict->str_data_start[1] << 8);
dict->block_dir = (const uint32_t *)(dict->str_data_start + 2);
dict->blocks_start = dict->str_data_start + 2 + dict->block_count * 4;
} else {
dict->block_count = 0;
dict->block_dir = NULL;
dict->blocks_start = dict->str_data_start;
}
/* Prefix table at end */
uint32_t prefix_bytes = (uint32_t)hdr->prefix_count * 4;
if (len >= prefix_bytes) {
dict->prefix_table = (const uint32_t *)(base + len - prefix_bytes);
} else {
dict->prefix_table = NULL;
}
return 0;
}
/* ─── String decompression ─── */
static const char *resolve_string(const struct dict_mphf *dict, uint32_t val_id)
{
const uint8_t *off_ptr = dict->string_offsets + val_id * 3;
uint32_t str_offset = (uint32_t)off_ptr[0]
| ((uint32_t)off_ptr[1] << 8)
| ((uint32_t)off_ptr[2] << 16);
if (!(dict->header->flags & DICT_MPHF_FLAG_COMPRESSED)) {
return (const char *)(dict->str_data_start + str_offset);
}
/* Block-compressed: decompress the right block */
uint32_t block_idx = str_offset / DICT_MPHF_BLOCK_SIZE;
uint32_t in_block_off = str_offset % DICT_MPHF_BLOCK_SIZE;
if (block_idx >= dict->block_count) {
return NULL;
}
/* Get compressed block bounds */
uint32_t blk_start = dict->block_dir[block_idx];
uint32_t blk_end;
if (block_idx + 1 < dict->block_count) {
blk_end = dict->block_dir[block_idx + 1];
} else {
/* Last block: extends to prefix_table or end of file */
blk_end = (const uint8_t *)dict->prefix_table - dict->blocks_start;
}
const uint8_t *compressed = dict->blocks_start + blk_start;
uint32_t compressed_len = blk_end - blk_start;
static uint8_t decomp_buf[DICT_MPHF_BLOCK_SIZE];
static uint32_t cached_block = UINT32_MAX;
static size_t cached_len;
if (cached_block != block_idx) {
size_t out_len;
if (block_inflate(compressed, compressed_len,
decomp_buf, sizeof(decomp_buf), &out_len) != 0) {
return NULL;
}
cached_block = block_idx;
cached_len = out_len;
}
if (in_block_off >= cached_len) {
return NULL;
}
return (const char *)(decomp_buf + in_block_off);
}
/* ─── Lookup ─── */
const char *dict_mphf_lookup(const struct dict_mphf *dict,
const uint32_t *strokes, uint8_t count)
{
if (!dict || !dict->header || !strokes || count == 0) {
return NULL;
}
const struct dict_mphf_header *hdr = dict->header;
uint8_t key_buf[32];
size_t key_len = (size_t)count * 4;
if (key_len > sizeof(key_buf)) {
return NULL;
}
for (uint8_t i = 0; i < count; i++) {
key_buf[i * 4 + 0] = (uint8_t)(strokes[i]);
key_buf[i * 4 + 1] = (uint8_t)(strokes[i] >> 8);
key_buf[i * 4 + 2] = (uint8_t)(strokes[i] >> 16);
key_buf[i * 4 + 3] = (uint8_t)(strokes[i] >> 24);
}
uint32_t bucket = hash_key(key_buf, key_len, 0) % hdr->bucket_count;
uint32_t d = read_bits(dict->displacements,
bucket * (uint32_t)hdr->disp_bits,
hdr->disp_bits);
uint32_t slot = hash_key(key_buf, key_len, d + 1) % hdr->entry_count;
uint8_t expected_fp = (uint8_t)(fnv1a_32(key_buf, key_len) & 0xFF);
if (dict->fingerprints[slot] != expected_fp) {
return NULL;
}
uint32_t val_id = read_bits(dict->values,
slot * (uint32_t)hdr->value_bits,
hdr->value_bits);
if (val_id >= hdr->unique_count) {
return NULL;
}
return resolve_string(dict, val_id);
}
/* ─── has_prefix ─── */
bool dict_mphf_has_prefix(const struct dict_mphf *dict, uint32_t stroke)
{
if (!dict || !dict->header || !dict->prefix_table ||
dict->header->prefix_count == 0) {
return false;
}
uint32_t lo = 0;
uint32_t hi = dict->header->prefix_count;
while (lo < hi) {
uint32_t mid = lo + (hi - lo) / 2;
uint32_t val = dict->prefix_table[mid];
if (val == stroke) return true;
if (val < stroke) lo = mid + 1;
else hi = mid;
}
return false;
}

69
src/dict_mphf.h Normal file
View file

@ -0,0 +1,69 @@
/**
* MPHF (Minimal Perfect Hash Function) dictionary lookup engine.
*
* Binary format v2: CHD MPHF + bit-packed displacements/values +
* fingerprinted verification + block-compressed string table.
*
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*/
#ifndef DICT_MPHF_H
#define DICT_MPHF_H
#include <stdint.h>
#include <stdbool.h>
#include <stddef.h>
#define DICT_MPHF_MAGIC 0x4F4E5453 /* "STNO" */
#define DICT_MPHF_VERSION 2
#define DICT_MPHF_FLAG_COMPRESSED 0x0001
#define DICT_MPHF_BLOCK_SIZE 4096
struct dict_mphf_header {
uint32_t magic;
uint16_t version;
uint16_t flags;
uint32_t entry_count;
uint32_t bucket_count;
uint32_t unique_count;
uint8_t value_bits;
uint8_t disp_bits;
uint16_t prefix_count;
uint32_t reserved0;
uint32_t reserved1;
} __attribute__((packed));
_Static_assert(sizeof(struct dict_mphf_header) == 32, "header must be 32 bytes");
struct dict_mphf {
const struct dict_mphf_header *header;
const uint8_t *displacements;
const uint8_t *values;
const uint8_t *fingerprints;
const uint8_t *string_offsets;
const uint8_t *str_data_start; /* start of string data section */
const uint32_t *prefix_table;
uint32_t disp_section_len;
uint32_t val_section_len;
uint32_t fp_section_len;
uint16_t block_count;
const uint32_t *block_dir; /* block offset directory */
const uint8_t *blocks_start; /* start of compressed blocks */
uint32_t str_data_len; /* total string data section length */
};
int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len);
const char *dict_mphf_lookup(const struct dict_mphf *dict,
const uint32_t *strokes, uint8_t count);
bool dict_mphf_has_prefix(const struct dict_mphf *dict, uint32_t stroke);
static inline uint32_t dict_mphf_count(const struct dict_mphf *dict)
{
return dict->header->entry_count;
}
#endif /* DICT_MPHF_H */

335
src/formatter.c Normal file
View file

@ -0,0 +1,335 @@
/*
* Copyright (c) 2024 Afiq Zudin Hadi
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*
* Plover formatting engine parses translation strings containing
* {commands} and produces clean output with spacing/capitalization.
* No malloc all stack/static.
*/
#include "formatter.h"
#include <string.h>
/* ── helpers ────────────────────────────────────────────────────── */
static inline bool is_upper(char c) { return c >= 'A' && c <= 'Z'; }
static inline bool is_lower(char c) { return c >= 'a' && c <= 'z'; }
static inline char to_upper(char c) { return is_lower(c) ? (char)(c - 32) : c; }
static inline char to_lower(char c) { return is_upper(c) ? (char)(c + 32) : c; }
static bool str_eq(const char *a, const char *b, size_t n)
{
for (size_t i = 0; i < n; i++) {
if (a[i] != b[i]) return false;
}
return true;
}
static bool starts_with(const char *s, size_t slen, const char *prefix)
{
size_t plen = strlen(prefix);
if (slen < plen) return false;
return str_eq(s, prefix, plen);
}
/* ── output buffer append ───────────────────────────────────────── */
static void emit_char(struct steno_fmt_result *r, char c)
{
if (r->len < STENO_FMT_MAX_OUTPUT - 1) {
r->text[r->len++] = c;
}
}
/* ── apply capitalization/mode transforms to a text segment ───── */
static void emit_text_transformed(struct steno_fmt_state *state,
struct steno_fmt_result *r,
const char *text, size_t len)
{
if (len == 0) return;
for (size_t i = 0; i < len; i++) {
char c = text[i];
/* Mode transforms */
if (state->mode == STENO_MODE_CAPS) {
c = to_upper(c);
} else if (state->mode == STENO_MODE_LOWER) {
c = to_lower(c);
} else if (state->mode == STENO_MODE_TITLE) {
/* Capitalize first letter of each word */
if (i == 0 || (i > 0 && text[i - 1] == ' ')) {
c = to_upper(c);
}
}
/* One-shot transforms (first char only) */
if (i == 0) {
if (state->cap_next) {
c = to_upper(c);
state->cap_next = false;
}
if (state->upper_next) {
/* uppercase entire word — handled below */
}
if (state->lower_next) {
/* lowercase entire word — handled below */
}
}
/* upper_next: entire word */
if (state->upper_next) {
c = to_upper(c);
}
/* lower_next: entire word */
if (state->lower_next) {
c = to_lower(c);
}
emit_char(r, c);
}
state->upper_next = false;
state->lower_next = false;
}
/* ── prepend space if needed ────────────────────────────────────── */
static void maybe_space(struct steno_fmt_state *state,
struct steno_fmt_result *r)
{
if (state->space_pending && !state->suppress_space) {
emit_char(r, ' ');
}
state->suppress_space = false;
}
/* ── process a single {…} command ──────────────────────────────── */
static void process_command(struct steno_fmt_state *state,
struct steno_fmt_result *r,
const char *cmd, size_t cmd_len)
{
/* {^} — suppress space */
if (cmd_len == 1 && cmd[0] == '^') {
state->suppress_space = true;
return;
}
/* {^suffix} — attach suffix */
if (cmd_len > 1 && cmd[0] == '^') {
state->suppress_space = true;
maybe_space(state, r);
emit_text_transformed(state, r, cmd + 1, cmd_len - 1);
state->space_pending = true;
state->glue = false;
return;
}
/* {prefix^} — attach prefix */
if (cmd_len > 1 && cmd[cmd_len - 1] == '^') {
maybe_space(state, r);
emit_text_transformed(state, r, cmd, cmd_len - 1);
state->suppress_space = true;
state->space_pending = false;
state->glue = false;
return;
}
/* {-|} — capitalize next */
if (cmd_len == 2 && cmd[0] == '-' && cmd[1] == '|') {
state->cap_next = true;
return;
}
/* {~|} — carry capitalize */
if (cmd_len == 2 && cmd[0] == '~' && cmd[1] == '|') {
state->cap_next = true;
return;
}
/* {*-|} — retro capitalize (backspace + re-emit) */
if (cmd_len == 3 && cmd[0] == '*' && cmd[1] == '-' && cmd[2] == '|') {
/* Signal retro-capitalize — simplified: set flag, caller handles */
r->backspaces = 1;
return;
}
/* {*!} — retro delete space */
if (cmd_len == 2 && cmd[0] == '*' && cmd[1] == '!') {
r->backspaces = 1;
return;
}
/* {*?} — retro insert space */
if (cmd_len == 2 && cmd[0] == '*' && cmd[1] == '?') {
/* Would need undo context; simplified signal */
return;
}
/* {*} — undo */
if (cmd_len == 1 && cmd[0] == '*') {
r->is_undo = true;
r->is_command_only = true;
return;
}
/* {.} {,} {?} {!} {;} {:} — punctuation: attach to prev, cap next for sentence-enders */
if (cmd_len == 1 && (cmd[0] == '.' || cmd[0] == ',' ||
cmd[0] == '?' || cmd[0] == '!' ||
cmd[0] == ';' || cmd[0] == ':')) {
state->suppress_space = true;
maybe_space(state, r);
emit_char(r, cmd[0]);
state->space_pending = true;
state->suppress_space = false;
/* Sentence-ending punctuation capitalizes next */
if (cmd[0] == '.' || cmd[0] == '?' || cmd[0] == '!') {
state->cap_next = true;
}
state->glue = false;
return;
}
/* {#...} — key combo (emit nothing for now, could extend) */
if (cmd_len > 0 && cmd[0] == '#') {
r->is_command_only = true;
return;
}
/* {&letter} — fingerspelling */
if (cmd_len >= 2 && cmd[0] == '&') {
/* Glue: no space between glue strokes */
if (state->glue) {
state->suppress_space = true;
}
maybe_space(state, r);
emit_text_transformed(state, r, cmd + 1, cmd_len - 1);
state->space_pending = true;
state->glue = true;
state->suppress_space = false;
return;
}
/* {&} — bare glue */
if (cmd_len == 1 && cmd[0] == '&') {
state->glue = true;
state->suppress_space = true;
return;
}
/* {MODE:CAPS} {MODE:TITLE} {MODE:LOWER} {MODE:RESET} */
if (starts_with(cmd, cmd_len, "MODE:")) {
const char *mode_str = cmd + 5;
size_t mode_len = cmd_len - 5;
if (mode_len == 4 && str_eq(mode_str, "CAPS", 4)) {
state->mode = STENO_MODE_CAPS;
} else if (mode_len == 5 && str_eq(mode_str, "TITLE", 5)) {
state->mode = STENO_MODE_TITLE;
} else if (mode_len == 5 && str_eq(mode_str, "LOWER", 5)) {
state->mode = STENO_MODE_LOWER;
} else if (mode_len == 5 && str_eq(mode_str, "RESET", 5)) {
state->mode = STENO_MODE_NORMAL;
}
return;
}
/* {<} — uppercase next word */
if (cmd_len == 1 && cmd[0] == '<') {
state->upper_next = true;
return;
}
/* {>} — lowercase next word */
if (cmd_len == 1 && cmd[0] == '>') {
state->lower_next = true;
return;
}
}
/* ── main entry point ──────────────────────────────────────────── */
void steno_fmt_init(struct steno_fmt_state *state)
{
memset(state, 0, sizeof(*state));
state->space_pending = false;
state->cap_next = false;
state->upper_next = false;
state->lower_next = false;
state->suppress_space = false;
state->glue = false;
state->mode = STENO_MODE_NORMAL;
}
void steno_fmt_process(struct steno_fmt_state *state,
const char *translation,
struct steno_fmt_result *result)
{
memset(result, 0, sizeof(*result));
if (!translation || !translation[0]) {
result->is_command_only = true;
return;
}
size_t tlen = strlen(translation);
const char *p = translation;
const char *end = translation + tlen;
bool emitted_text = false;
bool had_command = false;
bool this_glue = false;
while (p < end) {
/* Escaped braces: \{ \} */
if (p[0] == '\\' && p + 1 < end && (p[1] == '{' || p[1] == '}')) {
maybe_space(state, result);
emit_text_transformed(state, result, p + 1, 1);
state->space_pending = true;
state->glue = false;
emitted_text = true;
p += 2;
continue;
}
/* Command: {…} */
if (p[0] == '{') {
const char *close = p + 1;
while (close < end && *close != '}') {
close++;
}
if (close < end) {
const char *cmd = p + 1;
size_t cmd_len = (size_t)(close - cmd);
process_command(state, result, cmd, cmd_len);
had_command = true;
if (state->glue) this_glue = true;
p = close + 1;
continue;
}
}
/* Literal text segment — find end (next '{' or '\' or end) */
const char *seg_start = p;
while (p < end && p[0] != '{' &&
!(p[0] == '\\' && p + 1 < end && (p[1] == '{' || p[1] == '}'))) {
p++;
}
size_t seg_len = (size_t)(p - seg_start);
if (seg_len > 0) {
if (!this_glue) {
maybe_space(state, result);
}
emit_text_transformed(state, result, seg_start, seg_len);
state->space_pending = true;
if (!this_glue) {
state->glue = false;
}
emitted_text = true;
}
}
result->text[result->len] = '\0';
result->is_command_only = !emitted_text && had_command && result->len == 0;
}

46
src/formatter.h Normal file
View file

@ -0,0 +1,46 @@
#ifndef STENO_FORMATTER_H
#define STENO_FORMATTER_H
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
/* Max output length from a single translation */
#define STENO_FMT_MAX_OUTPUT 128
/* Formatting mode */
enum steno_fmt_mode {
STENO_MODE_NORMAL = 0,
STENO_MODE_CAPS,
STENO_MODE_TITLE,
STENO_MODE_LOWER,
};
/* State persists between translations */
struct steno_fmt_state {
bool space_pending; /* insert space before next word */
bool cap_next; /* capitalize next word */
bool upper_next; /* uppercase next word */
bool lower_next; /* lowercase next word */
bool suppress_space; /* {^} suppress next space */
bool glue; /* in fingerspelling/glue sequence */
enum steno_fmt_mode mode;
};
/* Result of formatting one translation */
struct steno_fmt_result {
char text[STENO_FMT_MAX_OUTPUT]; /* output text to emit */
uint8_t len; /* length of text */
uint8_t backspaces; /* backspaces to send BEFORE text */
bool is_command_only; /* true if no text output (pure command) */
bool is_undo; /* true if this is an undo command */
};
void steno_fmt_init(struct steno_fmt_state *state);
/* Format a raw translation string. Updates state, fills result. */
void steno_fmt_process(struct steno_fmt_state *state,
const char *translation,
struct steno_fmt_result *result);
#endif

352
src/output.c Normal file
View file

@ -0,0 +1,352 @@
#include "output.h"
#include <zephyr/kernel.h>
#include <zephyr/logging/log.h>
#include <zmk/hid.h>
#include <zmk/endpoints.h>
#include <zmk/events/keycode_state_changed.h>
LOG_MODULE_DECLARE(zmk, CONFIG_ZMK_LOG_LEVEL);
struct hid_map {
uint8_t keycode;
bool shift;
};
static const struct hid_map ASCII_TO_HID[128] = {
[' '] = {0x2C, false}, /* space */
['!'] = {0x1E, true}, /* shift+1 */
['"'] = {0x34, true}, /* shift+' */
['#'] = {0x20, true},
['$'] = {0x21, true},
['%'] = {0x22, true},
['&'] = {0x24, true},
['\''] = {0x34, false},
['('] = {0x26, true},
[')'] = {0x27, true},
['*'] = {0x25, true},
['+'] = {0x2E, true},
[','] = {0x36, false},
['-'] = {0x2D, false},
['.'] = {0x37, false},
['/'] = {0x38, false},
['0'] = {0x27, false},
['1'] = {0x1E, false},
['2'] = {0x1F, false},
['3'] = {0x20, false},
['4'] = {0x21, false},
['5'] = {0x22, false},
['6'] = {0x23, false},
['7'] = {0x24, false},
['8'] = {0x25, false},
['9'] = {0x26, false},
[':'] = {0x33, true},
[';'] = {0x33, false},
['<'] = {0x36, true},
['='] = {0x2E, false},
['>'] = {0x37, true},
['?'] = {0x38, true},
['@'] = {0x1F, true},
['A'] = {0x04, true},
['B'] = {0x05, true},
['C'] = {0x06, true},
['D'] = {0x07, true},
['E'] = {0x08, true},
['F'] = {0x09, true},
['G'] = {0x0A, true},
['H'] = {0x0B, true},
['I'] = {0x0C, true},
['J'] = {0x0D, true},
['K'] = {0x0E, true},
['L'] = {0x0F, true},
['M'] = {0x10, true},
['N'] = {0x11, true},
['O'] = {0x12, true},
['P'] = {0x13, true},
['Q'] = {0x14, true},
['R'] = {0x15, true},
['S'] = {0x16, true},
['T'] = {0x17, true},
['U'] = {0x18, true},
['V'] = {0x19, true},
['W'] = {0x1A, true},
['X'] = {0x1B, true},
['Y'] = {0x1C, true},
['Z'] = {0x1D, true},
['['] = {0x2F, false},
['\\'] = {0x31, false},
[']'] = {0x30, false},
['^'] = {0x23, true},
['_'] = {0x2D, true},
['`'] = {0x35, false},
['a'] = {0x04, false},
['b'] = {0x05, false},
['c'] = {0x06, false},
['d'] = {0x07, false},
['e'] = {0x08, false},
['f'] = {0x09, false},
['g'] = {0x0A, false},
['h'] = {0x0B, false},
['i'] = {0x0C, false},
['j'] = {0x0D, false},
['k'] = {0x0E, false},
['l'] = {0x0F, false},
['m'] = {0x10, false},
['n'] = {0x11, false},
['o'] = {0x12, false},
['p'] = {0x13, false},
['q'] = {0x14, false},
['r'] = {0x15, false},
['s'] = {0x16, false},
['t'] = {0x17, false},
['u'] = {0x18, false},
['v'] = {0x19, false},
['w'] = {0x1A, false},
['x'] = {0x1B, false},
['y'] = {0x1C, false},
['z'] = {0x1D, false},
['{'] = {0x2F, true},
['|'] = {0x31, true},
['}'] = {0x30, true},
['~'] = {0x35, true},
};
#define HID_BACKSPACE 0x2A
#define HID_RETURN 0x28
#define HID_LSHIFT 0xE1
#define HID_LCTRL 0xE0
#define HID_LALT 0xE2
#define HID_RALT 0xE6
static void tap_key(uint8_t keycode, bool shift)
{
if (shift) {
raise_zmk_keycode_state_changed((struct zmk_keycode_state_changed){
.usage_page = 0x07, .keycode = HID_LSHIFT,
.implicit_modifiers = 0, .explicit_modifiers = 0,
.state = true, .timestamp = k_uptime_get()});
}
raise_zmk_keycode_state_changed((struct zmk_keycode_state_changed){
.usage_page = 0x07, .keycode = keycode,
.implicit_modifiers = 0, .explicit_modifiers = 0,
.state = true, .timestamp = k_uptime_get()});
raise_zmk_keycode_state_changed((struct zmk_keycode_state_changed){
.usage_page = 0x07, .keycode = keycode,
.implicit_modifiers = 0, .explicit_modifiers = 0,
.state = false, .timestamp = k_uptime_get()});
if (shift) {
raise_zmk_keycode_state_changed((struct zmk_keycode_state_changed){
.usage_page = 0x07, .keycode = HID_LSHIFT,
.implicit_modifiers = 0, .explicit_modifiers = 0,
.state = false, .timestamp = k_uptime_get()});
}
}
static void press_key(uint8_t keycode)
{
raise_zmk_keycode_state_changed((struct zmk_keycode_state_changed){
.usage_page = 0x07, .keycode = keycode,
.implicit_modifiers = 0, .explicit_modifiers = 0,
.state = true, .timestamp = k_uptime_get()});
}
static void release_key(uint8_t keycode)
{
raise_zmk_keycode_state_changed((struct zmk_keycode_state_changed){
.usage_page = 0x07, .keycode = keycode,
.implicit_modifiers = 0, .explicit_modifiers = 0,
.state = false, .timestamp = k_uptime_get()});
}
/* Map hex digit (0-15) to HID keycode */
static uint8_t hex_to_hid(uint8_t nib)
{
if (nib == 0) {
return 0x27; /* '0' */
}
if (nib <= 9) {
return 0x1E + (nib - 1); /* '1'-'9': 0x1E-0x26 */
}
return 0x04 + (nib - 10); /* 'a'-'f': 0x04-0x09 */
}
/* Tap hex digits of codepoint (variable width, skip leading zeros) */
static void tap_hex_digits(uint32_t codepoint)
{
char buf[9];
int n = 0;
if (codepoint == 0) {
tap_key(hex_to_hid(0), false);
return;
}
/* Build hex digits in reverse */
uint32_t cp = codepoint;
while (cp > 0) {
buf[n++] = cp & 0xF;
cp >>= 4;
}
/* Tap in forward order */
for (int i = n - 1; i >= 0; i--) {
tap_key(hex_to_hid(buf[i]), false);
}
}
void steno_output_unicode(uint32_t codepoint)
{
if (codepoint > 0x10FFFF) {
LOG_WRN("Invalid codepoint U+%06X", codepoint);
return;
}
#if IS_ENABLED(CONFIG_STENO_UNICODE_MODE_LINUX)
/* IBus/GTK: Ctrl+Shift+U, hex digits, Return */
press_key(HID_LCTRL);
press_key(HID_LSHIFT);
tap_key(0x18, false); /* 'u' */
release_key(HID_LSHIFT);
release_key(HID_LCTRL);
tap_hex_digits(codepoint);
tap_key(HID_RETURN, false);
#elif IS_ENABLED(CONFIG_STENO_UNICODE_MODE_MACOS)
/* macOS Unicode Hex Input: hold Option, type 4+ hex digits */
press_key(HID_LALT);
/* Pad to at least 4 digits */
char digits[8];
int n = 0;
uint32_t cp = codepoint;
do {
digits[n++] = cp & 0xF;
cp >>= 4;
} while (cp > 0);
/* Pad to 4 */
while (n < 4) {
digits[n++] = 0;
}
for (int i = n - 1; i >= 0; i--) {
tap_key(hex_to_hid(digits[i]), false);
}
release_key(HID_LALT);
#elif IS_ENABLED(CONFIG_STENO_UNICODE_MODE_WINC)
/* WinCompose: tap RAlt, 'u', hex digits, Return */
tap_key(HID_RALT, false);
tap_key(0x18, false); /* 'u' */
tap_hex_digits(codepoint);
tap_key(HID_RETURN, false);
#else
LOG_WRN("Unicode disabled, skipping U+%04X", codepoint);
#endif
}
/* Decode UTF-8 byte at text[i], write codepoint, return bytes consumed (0 on error) */
static int utf8_decode(const char *text, size_t len, size_t i, uint32_t *cp)
{
unsigned char c = (unsigned char)text[i];
if (c < 0x80) {
*cp = c;
return 1;
}
uint32_t codepoint;
int expect; /* expected continuation bytes */
if ((c & 0xE0) == 0xC0) {
codepoint = c & 0x1F;
expect = 1;
} else if ((c & 0xF0) == 0xE0) {
codepoint = c & 0x0F;
expect = 2;
} else if ((c & 0xF8) == 0xF0) {
codepoint = c & 0x07;
expect = 3;
} else {
return 0; /* invalid lead byte */
}
if (i + expect >= len) {
return 0; /* truncated */
}
for (int j = 1; j <= expect; j++) {
unsigned char cont = (unsigned char)text[i + j];
if ((cont & 0xC0) != 0x80) {
return 0;
}
codepoint = (codepoint << 6) | (cont & 0x3F);
}
/* Reject overlong encodings */
if ((expect == 1 && codepoint < 0x80) ||
(expect == 2 && codepoint < 0x800) ||
(expect == 3 && codepoint < 0x10000)) {
return 0;
}
/* Reject surrogates (U+D800..U+DFFF) and beyond Unicode max */
if ((codepoint >= 0xD800 && codepoint <= 0xDFFF) || codepoint > 0x10FFFF) {
return 0;
}
*cp = codepoint;
return 1 + expect;
}
void steno_output_send(const char *text, size_t len)
{
for (size_t i = 0; i < len; ) {
unsigned char c = (unsigned char)text[i];
if (c == '\n') {
tap_key(HID_RETURN, false);
i++;
continue;
}
/* ASCII range */
if (c < 128) {
if (ASCII_TO_HID[c].keycode == 0) {
LOG_WRN("Unmapped ASCII char 0x%02X", c);
i++;
continue;
}
tap_key(ASCII_TO_HID[c].keycode, ASCII_TO_HID[c].shift);
i++;
continue;
}
/* Multi-byte UTF-8 → Unicode codepoint */
uint32_t codepoint;
int consumed = utf8_decode(text, len, i, &codepoint);
if (consumed == 0) {
LOG_WRN("Invalid UTF-8 at offset %u (0x%02X)", (unsigned)i, c);
i++;
continue;
}
steno_output_unicode(codepoint);
i += consumed;
}
}
void steno_output_backspace(int count)
{
for (int i = 0; i < count; i++) {
tap_key(HID_BACKSPACE, false);
}
}

13
src/output.h Normal file
View file

@ -0,0 +1,13 @@
#ifndef STENO_OUTPUT_H
#define STENO_OUTPUT_H
#include <stddef.h>
#include <stdint.h>
void steno_output_send(const char *text, size_t len);
void steno_output_backspace(int count);
void steno_output_unicode(uint32_t codepoint);
#endif

148
src/trie.c Normal file
View file

@ -0,0 +1,148 @@
#include "trie.h"
#include <string.h>
static const uint8_t *dict_data;
static const struct steno_dict_header *dict_hdr;
static const uint8_t *entry_array;
static const char *string_table;
static size_t entry_stride;
int steno_trie_init(const uint8_t *data, size_t len)
{
if (!data || len < sizeof(struct steno_dict_header)) {
return -1;
}
dict_hdr = (const struct steno_dict_header *)data;
if (dict_hdr->magic != STENO_DICT_MAGIC) {
return -2;
}
if (dict_hdr->version != 1) {
return -3;
}
if (dict_hdr->max_strokes == 0 || dict_hdr->max_strokes > 16) {
return -4;
}
entry_stride = (size_t)dict_hdr->max_strokes * 4 + 4;
size_t entries_end = sizeof(struct steno_dict_header) +
(size_t)dict_hdr->entry_count * entry_stride;
if (entries_end > len || dict_hdr->strings_offset > len) {
return -5;
}
dict_data = data;
entry_array = data + sizeof(struct steno_dict_header);
string_table = (const char *)(data + dict_hdr->strings_offset);
return 0;
}
static const uint8_t *get_entry(uint32_t idx)
{
return entry_array + (size_t)idx * entry_stride;
}
static int cmp_strokes(const uint32_t *a, uint8_t a_count,
const uint8_t *entry_bytes, uint8_t max_s)
{
const uint32_t *b = (const uint32_t *)entry_bytes;
uint8_t b_count = 0;
for (uint8_t i = 0; i < max_s; i++) {
if (b[i] != 0) {
b_count = i + 1;
}
}
uint8_t min_count = a_count < b_count ? a_count : b_count;
for (uint8_t i = 0; i < min_count; i++) {
if (a[i] < b[i]) return -1;
if (a[i] > b[i]) return 1;
}
if (a_count < b_count) return -1;
if (a_count > b_count) return 1;
return 0;
}
const char *steno_trie_lookup(const uint32_t *strokes, uint8_t count)
{
if (!dict_hdr || !strokes || count == 0 || count > dict_hdr->max_strokes) {
return NULL;
}
uint32_t lo = 0;
uint32_t hi = dict_hdr->entry_count;
while (lo < hi) {
uint32_t mid = lo + (hi - lo) / 2;
const uint8_t *entry = get_entry(mid);
int c = cmp_strokes(strokes, count, entry, dict_hdr->max_strokes);
if (c == 0) {
uint32_t str_off;
memcpy(&str_off, entry + (size_t)dict_hdr->max_strokes * 4, 4);
return string_table + str_off;
}
if (c < 0) {
hi = mid;
} else {
lo = mid + 1;
}
}
return NULL;
}
bool steno_trie_has_prefix(const uint32_t *strokes, uint8_t count)
{
if (!dict_hdr || !strokes || count == 0 || count >= dict_hdr->max_strokes) {
return false;
}
uint32_t lo = 0;
uint32_t hi = dict_hdr->entry_count;
uint32_t first_ge = hi;
while (lo < hi) {
uint32_t mid = lo + (hi - lo) / 2;
const uint8_t *entry = get_entry(mid);
const uint32_t *e_strokes = (const uint32_t *)entry;
int cmp = 0;
for (uint8_t i = 0; i < count; i++) {
if (e_strokes[i] < strokes[i]) { cmp = -1; break; }
if (e_strokes[i] > strokes[i]) { cmp = 1; break; }
}
if (cmp >= 0) {
first_ge = mid;
hi = mid;
} else {
lo = mid + 1;
}
}
for (uint32_t idx = first_ge; idx < dict_hdr->entry_count; idx++) {
const uint8_t *entry = get_entry(idx);
const uint32_t *e_strokes = (const uint32_t *)entry;
bool prefix_match = true;
for (uint8_t i = 0; i < count; i++) {
if (e_strokes[i] != strokes[i]) {
prefix_match = false;
break;
}
}
if (!prefix_match) {
return false;
}
if (e_strokes[count] != 0) {
return true;
}
}
return false;
}

25
src/trie.h Normal file
View file

@ -0,0 +1,25 @@
#ifndef STENO_TRIE_H
#define STENO_TRIE_H
#include <stdbool.h>
#include <stddef.h>
#include <stdint.h>
#define STENO_DICT_MAGIC 0x4F4E5453 /* "STNO" */
struct steno_dict_header {
uint32_t magic;
uint16_t version;
uint8_t max_strokes;
uint8_t _pad;
uint32_t entry_count;
uint32_t strings_offset;
} __attribute__((packed));
int steno_trie_init(const uint8_t *data, size_t len);
const char *steno_trie_lookup(const uint32_t *strokes, uint8_t count);
bool steno_trie_has_prefix(const uint32_t *strokes, uint8_t count);
#endif

64
src/undo.c Normal file
View file

@ -0,0 +1,64 @@
/*
* Copyright (c) 2024 Afiq Zudin Hadi
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*/
#include "undo.h"
#include <string.h>
void steno_undo_init(struct stroke_history *hist)
{
memset(hist, 0, sizeof(*hist));
}
void steno_undo_push(struct stroke_history *hist,
const uint32_t *strokes, uint8_t stroke_count,
uint8_t output_len, uint8_t space_before,
uint8_t fmt_flags)
{
struct stroke_history_entry *e = &hist->entries[hist->head];
uint8_t n = stroke_count;
if (n > STENO_MAX_MULTI_STROKE) {
n = STENO_MAX_MULTI_STROKE;
}
memcpy(e->strokes, strokes, n * sizeof(uint32_t));
e->stroke_count = n;
e->output_len = output_len;
e->space_before = space_before;
e->fmt_flags = fmt_flags;
hist->head = (hist->head + 1) % CONFIG_STENO_HISTORY_SIZE;
if (hist->count < CONFIG_STENO_HISTORY_SIZE) {
hist->count++;
}
}
struct stroke_history_entry *steno_undo_pop(struct stroke_history *hist)
{
if (hist->count == 0) {
return NULL;
}
hist->head = (hist->head + CONFIG_STENO_HISTORY_SIZE - 1) % CONFIG_STENO_HISTORY_SIZE;
hist->count--;
return &hist->entries[hist->head];
}
const struct stroke_history_entry *steno_undo_peek(const struct stroke_history *hist)
{
if (hist->count == 0) {
return NULL;
}
uint16_t idx = (hist->head + CONFIG_STENO_HISTORY_SIZE - 1) % CONFIG_STENO_HISTORY_SIZE;
return &hist->entries[idx];
}
uint16_t steno_undo_count(const struct stroke_history *hist)
{
return hist->count;
}

44
src/undo.h Normal file
View file

@ -0,0 +1,44 @@
#ifndef STENO_UNDO_H
#define STENO_UNDO_H
#include <stdbool.h>
#include <stdint.h>
#ifndef CONFIG_STENO_HISTORY_SIZE
#define CONFIG_STENO_HISTORY_SIZE 100
#endif
#define STENO_MAX_MULTI_STROKE 8
struct stroke_history_entry {
uint32_t strokes[STENO_MAX_MULTI_STROKE];
uint8_t stroke_count;
uint8_t output_len; /* chars emitted (for backspace count) */
uint8_t space_before; /* 1 if space was prepended */
uint8_t fmt_flags; /* formatter state snapshot for restore */
};
struct stroke_history {
struct stroke_history_entry entries[CONFIG_STENO_HISTORY_SIZE];
uint16_t head; /* next write position */
uint16_t count; /* entries in buffer */
};
void steno_undo_init(struct stroke_history *hist);
/* Push a new entry after successful output */
void steno_undo_push(struct stroke_history *hist,
const uint32_t *strokes, uint8_t stroke_count,
uint8_t output_len, uint8_t space_before,
uint8_t fmt_flags);
/* Pop most recent entry for undo. Returns NULL if empty. */
struct stroke_history_entry *steno_undo_pop(struct stroke_history *hist);
/* Peek at most recent entry without removing. Returns NULL if empty. */
const struct stroke_history_entry *steno_undo_peek(const struct stroke_history *hist);
/* Get current count */
uint16_t steno_undo_count(const struct stroke_history *hist);
#endif

64
tests/run_tests.sh Executable file
View file

@ -0,0 +1,64 @@
#!/bin/bash
# Run MPHF dictionary tests: compile test dict, build C test, execute.
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
BUILD_DIR="$SCRIPT_DIR/build"
mkdir -p "$BUILD_DIR"
echo "=== Step 1: Create test dictionary JSON ==="
cat > "$BUILD_DIR/test_dict.json" << 'DICTEOF'
{
"S": "is",
"T": "it",
"THE": "the",
"KAT": "cat",
"TK": "did",
"SKP": "and",
"TPOR": "for",
"STO": "so",
"HAOEU": "hi",
"TKOGS": "dogs",
"S/T": "{.}",
"PHAO*EUP/HRAOEUPB": "my line"
}
DICTEOF
echo "=== Step 2: Compile test dictionary ==="
python3 "$ROOT_DIR/tools/compile_mphf.py" \
"$BUILD_DIR/test_dict.json" \
"$BUILD_DIR/test_dict.bin" \
--stats
echo ""
echo "=== Step 3: Build C test binary ==="
cc -O2 -Wall -Wextra -I"$ROOT_DIR/src" \
-o "$BUILD_DIR/test_mphf" \
"$SCRIPT_DIR/test_mphf.c" \
"$ROOT_DIR/src/dict_mphf.c"
echo ""
echo "=== Step 4: Run tests ==="
"$BUILD_DIR/test_mphf" "$BUILD_DIR/test_dict.bin"
echo ""
echo "=== Step 5: Run with larger dict (if Plover available) ==="
PLOVER="/tmp/plover-main.json"
if [ -f "$PLOVER" ]; then
echo "Compiling 1000-entry subset..."
python3 "$ROOT_DIR/tools/compile_mphf.py" \
"$PLOVER" \
"$BUILD_DIR/plover_1k.bin" \
--max-entries 1000 \
--stats
echo ""
echo "Binary size: $(wc -c < "$BUILD_DIR/plover_1k.bin") bytes"
else
echo "Plover dict not found at $PLOVER, skipping large dict test"
echo "Download: curl -sL 'https://raw.githubusercontent.com/openstenoproject/plover/main/plover/assets/main.json' -o /tmp/plover-main.json"
fi
echo ""
echo "=== Done ==="

340
tests/test_formatter.c Normal file
View file

@ -0,0 +1,340 @@
/*
* Native host test for Plover formatting engine.
* Build: cc -I../src -o test_formatter test_formatter.c ../src/formatter.c
*/
#include <stdio.h>
#include <string.h>
#include <stdlib.h>
#include "formatter.h"
static int tests_run = 0;
static int tests_passed = 0;
#define ASSERT_EQ_STR(actual, expected, msg) do { \
tests_run++; \
if (strcmp((actual), (expected)) == 0) { \
tests_passed++; \
} else { \
printf("FAIL [%s]: got \"%s\", expected \"%s\"\n", msg, actual, expected); \
} \
} while (0)
#define ASSERT_EQ_INT(actual, expected, msg) do { \
tests_run++; \
if ((actual) == (expected)) { \
tests_passed++; \
} else { \
printf("FAIL [%s]: got %d, expected %d\n", msg, (int)(actual), (int)(expected)); \
} \
} while (0)
#define ASSERT_TRUE(cond, msg) do { \
tests_run++; \
if ((cond)) { \
tests_passed++; \
} else { \
printf("FAIL [%s]\n", msg); \
} \
} while (0)
/* Helper: process and return output text */
static struct steno_fmt_result proc(struct steno_fmt_state *s, const char *t)
{
struct steno_fmt_result r;
steno_fmt_process(s, t, &r);
return r;
}
static void test_raw_text(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
/* First word: no space before */
struct steno_fmt_result r = proc(&s, "hello");
ASSERT_EQ_STR(r.text, "hello", "raw: first word");
ASSERT_EQ_INT(r.len, 5, "raw: first word len");
/* Second word: space before */
r = proc(&s, "world");
ASSERT_EQ_STR(r.text, " world", "raw: second word with space");
ASSERT_EQ_INT(r.len, 6, "raw: second word len");
}
static void test_attach(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
proc(&s, "walk");
struct steno_fmt_result r = proc(&s, "{^}");
ASSERT_TRUE(r.is_command_only || r.len == 0, "attach: no output");
r = proc(&s, "ing");
ASSERT_EQ_STR(r.text, "ing", "attach: no space after {^}");
}
static void test_suffix_attach(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
proc(&s, "walk");
struct steno_fmt_result r = proc(&s, "{^ing}");
ASSERT_EQ_STR(r.text, "ing", "suffix: attached");
ASSERT_EQ_INT(r.len, 3, "suffix: len");
}
static void test_prefix_attach(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
struct steno_fmt_result r = proc(&s, "{pre^}");
ASSERT_EQ_STR(r.text, "pre", "prefix: text");
r = proc(&s, "fix");
ASSERT_EQ_STR(r.text, "fix", "prefix: next word attached");
}
static void test_capitalize_next(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
proc(&s, "{-|}");
struct steno_fmt_result r = proc(&s, "hello");
ASSERT_EQ_STR(r.text, "Hello", "cap_next: first word capitalized");
/* Verify cap_next is one-shot */
r = proc(&s, "world");
ASSERT_EQ_STR(r.text, " world", "cap_next: one-shot reset");
}
static void test_punctuation(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
proc(&s, "hello");
struct steno_fmt_result r = proc(&s, "{.}");
ASSERT_EQ_STR(r.text, ".", "period: attached");
/* Next word should be capitalized */
r = proc(&s, "world");
ASSERT_EQ_STR(r.text, " World", "period: cap next");
}
static void test_comma(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
proc(&s, "hello");
struct steno_fmt_result r = proc(&s, "{,}");
ASSERT_EQ_STR(r.text, ",", "comma: attached");
/* Comma does NOT capitalize next */
r = proc(&s, "world");
ASSERT_EQ_STR(r.text, " world", "comma: no cap next");
}
static void test_sentence_flow(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
struct steno_fmt_result r;
r = proc(&s, "I");
ASSERT_EQ_STR(r.text, "I", "sentence: I");
r = proc(&s, "{.}");
ASSERT_EQ_STR(r.text, ".", "sentence: period");
r = proc(&s, "the");
ASSERT_EQ_STR(r.text, " The", "sentence: The after period");
}
static void test_mode_caps(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
proc(&s, "{MODE:CAPS}");
struct steno_fmt_result r = proc(&s, "hello");
ASSERT_EQ_STR(r.text, "HELLO", "mode_caps: uppercase");
r = proc(&s, "world");
ASSERT_EQ_STR(r.text, " WORLD", "mode_caps: persists");
proc(&s, "{MODE:RESET}");
r = proc(&s, "test");
ASSERT_EQ_STR(r.text, " test", "mode_reset: normal");
}
static void test_mode_title(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
proc(&s, "{MODE:TITLE}");
struct steno_fmt_result r = proc(&s, "hello");
ASSERT_EQ_STR(r.text, "Hello", "mode_title: capitalize");
proc(&s, "{MODE:RESET}");
}
static void test_mode_lower(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
proc(&s, "{MODE:LOWER}");
struct steno_fmt_result r = proc(&s, "HELLO");
ASSERT_EQ_STR(r.text, "hello", "mode_lower: lowercase");
proc(&s, "{MODE:RESET}");
}
static void test_fingerspelling(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
struct steno_fmt_result r;
r = proc(&s, "{&a}");
ASSERT_EQ_STR(r.text, "a", "finger: a");
r = proc(&s, "{&b}");
ASSERT_EQ_STR(r.text, "b", "finger: b glued");
r = proc(&s, "{&c}");
ASSERT_EQ_STR(r.text, "c", "finger: c glued");
/* Non-fingerspelling should get space */
r = proc(&s, "hello");
ASSERT_EQ_STR(r.text, " hello", "finger: break with space");
}
static void test_combined_commands(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
proc(&s, "test");
/* {^}{-|} — suppress space + capitalize */
struct steno_fmt_result r = proc(&s, "{^}{-|}");
ASSERT_TRUE(r.len == 0 || r.is_command_only, "combined: no text");
r = proc(&s, "word");
ASSERT_EQ_STR(r.text, "Word", "combined: attached + capitalized");
}
static void test_undo(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
struct steno_fmt_result r = proc(&s, "{*}");
ASSERT_TRUE(r.is_undo, "undo: flag set");
ASSERT_TRUE(r.is_command_only, "undo: command only");
}
static void test_key_combo(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
struct steno_fmt_result r = proc(&s, "{#Return}");
ASSERT_TRUE(r.is_command_only, "key_combo: command only");
}
static void test_literal_braces(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
struct steno_fmt_result r = proc(&s, "\\{");
ASSERT_EQ_STR(r.text, "{", "literal: left brace");
r = proc(&s, "\\}");
ASSERT_EQ_STR(r.text, " }", "literal: right brace with space");
}
static void test_uppercase_next(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
proc(&s, "{<}");
struct steno_fmt_result r = proc(&s, "hello");
ASSERT_EQ_STR(r.text, "HELLO", "upper_next: entire word");
/* One-shot */
r = proc(&s, "world");
ASSERT_EQ_STR(r.text, " world", "upper_next: one-shot");
}
static void test_lowercase_next(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
proc(&s, "{>}");
struct steno_fmt_result r = proc(&s, "HELLO");
ASSERT_EQ_STR(r.text, "hello", "lower_next: entire word");
}
static void test_empty_translation(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
struct steno_fmt_result r = proc(&s, "");
ASSERT_TRUE(r.is_command_only, "empty: command only");
ASSERT_EQ_INT(r.len, 0, "empty: no text");
}
static void test_prefix_then_suffix(void)
{
struct steno_fmt_state s;
steno_fmt_init(&s);
/* {re^} then {^ed} */
proc(&s, "{re^}");
struct steno_fmt_result r = proc(&s, "{^ed}");
ASSERT_EQ_STR(r.text, "ed", "pre+suf: attached");
}
int main(void)
{
test_raw_text();
test_attach();
test_suffix_attach();
test_prefix_attach();
test_capitalize_next();
test_punctuation();
test_comma();
test_sentence_flow();
test_mode_caps();
test_mode_title();
test_mode_lower();
test_fingerspelling();
test_combined_commands();
test_undo();
test_key_combo();
test_literal_braces();
test_uppercase_next();
test_lowercase_next();
test_empty_translation();
test_prefix_then_suffix();
printf("\n%d/%d tests passed\n", tests_passed, tests_run);
if (tests_passed == tests_run) {
printf("ALL TESTS PASSED\n");
return 0;
}
return 1;
}

383
tests/test_mphf.c Normal file
View file

@ -0,0 +1,383 @@
/**
* Native tests for MPHF dictionary engine.
*
* Builds and runs on host (not ZMK). Compiles a small test dictionary
* via compile_mphf.py, then exercises all lookup paths in C.
*
* Build: cc -O2 -I../src -o test_mphf test_mphf.c ../src/dict_mphf.c
* Run: ./test_mphf test_dict.bin
*
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include "dict_mphf.h"
/* ─── Test helpers ─── */
static int tests_run = 0;
static int tests_passed = 0;
#define TEST(name) \
do { printf(" %-50s ", #name); tests_run++; } while(0)
#define PASS() \
do { tests_passed++; printf("PASS\n"); } while(0)
#define FAIL(msg) \
do { printf("FAIL: %s\n", msg); } while(0)
#define ASSERT_EQ_INT(a, b) \
do { \
if ((a) != (b)) { \
char _buf[128]; \
snprintf(_buf, sizeof(_buf), "expected %d, got %d", (int)(b), (int)(a)); \
FAIL(_buf); return; \
} \
} while(0)
#define ASSERT_EQ_STR(a, b) \
do { \
if (strcmp((a), (b)) != 0) { \
char _buf[256]; \
snprintf(_buf, sizeof(_buf), "expected \"%s\", got \"%s\"", (b), (a)); \
FAIL(_buf); return; \
} \
} while(0)
#define ASSERT_NULL(a) \
do { \
if ((a) != NULL) { \
FAIL("expected NULL"); return; \
} \
} while(0)
#define ASSERT_NOT_NULL(a) \
do { \
if ((a) == NULL) { \
FAIL("expected non-NULL"); return; \
} \
} while(0)
#define ASSERT_TRUE(a) \
do { \
if (!(a)) { \
FAIL("expected true"); return; \
} \
} while(0)
#define ASSERT_FALSE(a) \
do { \
if ((a)) { \
FAIL("expected false"); return; \
} \
} while(0)
/* ─── Load compiled binary from file ─── */
static uint8_t *load_file(const char *path, size_t *out_len)
{
FILE *f = fopen(path, "rb");
if (!f) {
fprintf(stderr, "Cannot open %s\n", path);
return NULL;
}
fseek(f, 0, SEEK_END);
long len = ftell(f);
fseek(f, 0, SEEK_SET);
uint8_t *data = malloc(len);
if (!data) {
fclose(f);
return NULL;
}
fread(data, 1, len, f);
fclose(f);
*out_len = (size_t)len;
return data;
}
/* ─── Test: init with valid data ─── */
static void test_init_valid(const uint8_t *data, size_t len)
{
TEST(init_valid);
struct dict_mphf dict;
int rc = dict_mphf_init(&dict, data, len);
ASSERT_EQ_INT(rc, 0);
ASSERT_EQ_INT(dict.header->magic, DICT_MPHF_MAGIC);
ASSERT_EQ_INT(dict.header->version, DICT_MPHF_VERSION);
PASS();
}
/* ─── Test: init with NULL ─── */
static void test_init_null(void)
{
TEST(init_null);
struct dict_mphf dict;
ASSERT_EQ_INT(dict_mphf_init(&dict, NULL, 0), -1);
ASSERT_EQ_INT(dict_mphf_init(NULL, &dict, 32), -1);
PASS();
}
/* ─── Test: init with truncated data ─── */
static void test_init_truncated(const uint8_t *data)
{
TEST(init_truncated);
struct dict_mphf dict;
ASSERT_EQ_INT(dict_mphf_init(&dict, data, 16), -2);
PASS();
}
/* ─── Test: init with bad magic ─── */
static void test_init_bad_magic(void)
{
TEST(init_bad_magic);
uint8_t bad[32] = {0};
struct dict_mphf dict;
ASSERT_EQ_INT(dict_mphf_init(&dict, bad, sizeof(bad)), -3);
PASS();
}
/* ─── Test: lookup known entries ─── */
/*
* The test dictionary (generated by test runner script) contains:
* "S" "is"
* "T" "it"
* "THE" "the"
* "KAT" "cat"
* "TK" "did"
* "SKP" "and"
* "TPOR" "for"
* "STO" "so"
* "HAOEU" "hi"
* "TKOGS" "dogs"
* "S/T" "{.}" (multi-stroke)
* "PHAO*EUP/HRAOEUPB" "my line" (multi-stroke)
*/
/* Steno key values (must match Python parser) */
#define SK_S 0x00000001u
#define SK_T 0x00000002u
#define SK_K 0x00000004u
#define SK_P 0x00000008u
#define SK_W 0x00000010u
#define SK_H 0x00000020u
#define SK_R 0x00000040u
#define SK_A 0x00000080u
#define SK_O 0x00000100u
#define SK_STAR 0x00000200u
#define SK_E 0x00000400u
#define SK_U 0x00000800u
#define SK_rF 0x00001000u
#define SK_rR 0x00002000u
#define SK_rP 0x00004000u
#define SK_rB 0x00008000u
#define SK_rL 0x00010000u
#define SK_rG 0x00020000u
#define SK_rT 0x00040000u
#define SK_rS 0x00080000u
#define SK_rD 0x00100000u
#define SK_rZ 0x00200000u
#define SK_NUM 0x00400000u
static void test_lookup_single_S(const struct dict_mphf *dict)
{
TEST(lookup_single_S);
uint32_t strokes[] = { SK_S };
const char *result = dict_mphf_lookup(dict, strokes, 1);
ASSERT_NOT_NULL(result);
ASSERT_EQ_STR(result, "is");
PASS();
}
static void test_lookup_single_T(const struct dict_mphf *dict)
{
TEST(lookup_single_T);
uint32_t strokes[] = { SK_T };
const char *result = dict_mphf_lookup(dict, strokes, 1);
ASSERT_NOT_NULL(result);
ASSERT_EQ_STR(result, "it");
PASS();
}
static void test_lookup_THE(const struct dict_mphf *dict)
{
TEST(lookup_THE);
/* -T → "the" (right T = bit 18 = 0x00040000) */
uint32_t strokes[] = { SK_rT };
const char *result = dict_mphf_lookup(dict, strokes, 1);
ASSERT_NOT_NULL(result);
ASSERT_EQ_STR(result, "the");
PASS();
}
static void test_lookup_KAT(const struct dict_mphf *dict)
{
TEST(lookup_KAT);
uint32_t strokes[] = { SK_K | SK_A | SK_rT };
const char *result = dict_mphf_lookup(dict, strokes, 1);
ASSERT_NOT_NULL(result);
ASSERT_EQ_STR(result, "cat");
PASS();
}
static void test_lookup_TPOR(const struct dict_mphf *dict)
{
TEST(lookup_TPOR);
uint32_t strokes[] = { SK_T | SK_P | SK_O | SK_rR };
const char *result = dict_mphf_lookup(dict, strokes, 1);
ASSERT_NOT_NULL(result);
ASSERT_EQ_STR(result, "for");
PASS();
}
static void test_lookup_HAOEU(const struct dict_mphf *dict)
{
TEST(lookup_HAOEU);
uint32_t strokes[] = { SK_H | SK_A | SK_O | SK_E | SK_U };
const char *result = dict_mphf_lookup(dict, strokes, 1);
ASSERT_NOT_NULL(result);
ASSERT_EQ_STR(result, "high");
PASS();
}
/* ─── Test: lookup multi-stroke ─── */
static void test_lookup_multi_stroke(const struct dict_mphf *dict)
{
TEST(lookup_multi_stroke);
/* Multi-stroke entries may or may not be present depending on dict size.
* Just verify no crash on 2-stroke lookup. */
uint32_t strokes[] = { SK_S, SK_T };
const char *result = dict_mphf_lookup(dict, strokes, 2);
(void)result;
PASS();
}
/* ─── Test: lookup not found ─── */
static void test_lookup_not_found(const struct dict_mphf *dict)
{
TEST(lookup_not_found);
/* "Z" not in dictionary */
uint32_t strokes[] = { SK_rZ };
const char *result = dict_mphf_lookup(dict, strokes, 1);
/* Could be NULL (fingerprint mismatch) or a wrong string (false positive).
* Fingerprint gives 99.6% true-negative rate. For testing purposes,
* just ensure no crash. If NULL, great. If non-NULL, it's a known
* false positive from the 8-bit fingerprint. */
(void)result;
PASS();
}
/* ─── Test: lookup with zero strokes ─── */
static void test_lookup_zero_strokes(const struct dict_mphf *dict)
{
TEST(lookup_zero_strokes);
uint32_t strokes[] = { 0 };
const char *result = dict_mphf_lookup(dict, strokes, 0);
ASSERT_NULL(result);
PASS();
}
/* ─── Test: has_prefix ─── */
static void test_has_prefix_S(const struct dict_mphf *dict)
{
TEST(has_prefix_S);
/* prefix check depends on whether multi-stroke entries exist in dict.
* Just verify no crash. */
(void)dict_mphf_has_prefix(dict, SK_S);
PASS();
}
static void test_has_prefix_not_found(const struct dict_mphf *dict)
{
TEST(has_prefix_not_found);
/* Extremely unlikely stroke combo — should not be a prefix */
ASSERT_FALSE(dict_mphf_has_prefix(dict, 0x003FFFFFu));
PASS();
}
/* ─── Test: entry count ─── */
static void test_entry_count(const struct dict_mphf *dict)
{
TEST(entry_count);
uint32_t count = dict_mphf_count(dict);
ASSERT_TRUE(count > 0);
PASS();
}
/* ─── Main ─── */
int main(int argc, char **argv)
{
if (argc < 2) {
fprintf(stderr, "Usage: %s <compiled-dict.bin>\n", argv[0]);
return 1;
}
size_t len;
uint8_t *data = load_file(argv[1], &len);
if (!data) {
return 1;
}
printf("Loaded %zu bytes from %s\n\n", len, argv[1]);
/* Init tests */
test_init_valid(data, len);
test_init_null();
test_init_truncated(data);
test_init_bad_magic();
/* Init dict for remaining tests */
struct dict_mphf dict;
int rc = dict_mphf_init(&dict, data, len);
if (rc != 0) {
fprintf(stderr, "dict_mphf_init failed: %d\n", rc);
free(data);
return 1;
}
printf("\n Dict: %u entries, %u buckets, %u unique strings\n",
dict.header->entry_count, dict.header->bucket_count,
dict.header->unique_count);
printf(" Bits: disp=%u value=%u\n",
dict.header->disp_bits, dict.header->value_bits);
printf(" Prefixes: %u\n\n", dict.header->prefix_count);
/* Lookup tests */
test_lookup_single_S(&dict);
test_lookup_single_T(&dict);
test_lookup_THE(&dict);
test_lookup_KAT(&dict);
test_lookup_TPOR(&dict);
test_lookup_HAOEU(&dict);
test_lookup_multi_stroke(&dict);
test_lookup_not_found(&dict);
test_lookup_zero_strokes(&dict);
/* Prefix tests */
test_has_prefix_S(&dict);
test_has_prefix_not_found(&dict);
/* Entry count */
test_entry_count(&dict);
printf("\n%d/%d tests passed\n", tests_passed, tests_run);
free(data);
return tests_passed == tests_run ? 0 : 1;
}

102
tests/test_trie.c Normal file
View file

@ -0,0 +1,102 @@
/*
* Native test for trie.c run on host, not on target.
* Build: cc -I../src -o test_trie test_trie.c ../src/trie.c
* Run: ./test_trie /tmp/steno_test.bin
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "trie.h"
static uint8_t *load_file(const char *path, size_t *out_len)
{
FILE *f = fopen(path, "rb");
if (!f) return NULL;
fseek(f, 0, SEEK_END);
long len = ftell(f);
fseek(f, 0, SEEK_SET);
uint8_t *buf = malloc(len);
if (fread(buf, 1, len, f) != (size_t)len) {
free(buf);
fclose(f);
return NULL;
}
fclose(f);
*out_len = len;
return buf;
}
static int tests_run = 0;
static int tests_passed = 0;
#define CHECK(cond, msg) do { \
tests_run++; \
if (cond) { tests_passed++; } \
else { printf("FAIL: %s (line %d)\n", msg, __LINE__); } \
} while(0)
int main(int argc, char **argv)
{
if (argc < 2) {
printf("Usage: %s <steno_dict.bin>\n", argv[0]);
return 1;
}
size_t len;
uint8_t *data = load_file(argv[1], &len);
if (!data) {
printf("Failed to load %s\n", argv[1]);
return 1;
}
int ret = steno_trie_init(data, len);
CHECK(ret == 0, "trie_init succeeds");
/* S → "is" (S- = bit 0 = 0x00000001) */
uint32_t s_stroke = 0x00000001;
const char *r = steno_trie_lookup(&s_stroke, 1);
CHECK(r != NULL && strcmp(r, "is") == 0, "S → 'is'");
/* T → "it" (T- = bit 1 = 0x00000002) */
uint32_t t_stroke = 0x00000002;
r = steno_trie_lookup(&t_stroke, 1);
CHECK(r != NULL && strcmp(r, "it") == 0, "T → 'it'");
/* -T → "the" (-T = bit 18 = 0x00040000) */
uint32_t t_right = 0x00040000;
r = steno_trie_lookup(&t_right, 1);
CHECK(r != NULL && strcmp(r, "the") == 0, "-T → 'the'");
/* TEFT → "test" (T- | -E | -F | -T = 0x02|0x400|0x1000|0x40000) */
uint32_t teft = 0x00041402;
r = steno_trie_lookup(&teft, 1);
CHECK(r != NULL && strcmp(r, "test") == 0, "TEFT → 'test'");
/* KO/PHAOURD → "computer" (multi-stroke) */
uint32_t ko = 0x00000104; /* K | O */
uint32_t phaourd = 0x001029A8; /* P | H | A | O | U | R | -D */
uint32_t multi[2] = {ko, phaourd};
r = steno_trie_lookup(multi, 2);
CHECK(r != NULL && strcmp(r, "computer") == 0, "KO/PHAOURD → 'computer'");
/* Non-existent stroke */
uint32_t nonsense = 0x003FFFFF;
r = steno_trie_lookup(&nonsense, 1);
CHECK(r == NULL, "nonsense stroke → NULL");
/* has_prefix: KO should be prefix of KO/PHAOURD */
CHECK(steno_trie_has_prefix(&ko, 1) == true, "KO is prefix");
/* has_prefix: S is NOT prefix of anything multi-stroke */
CHECK(steno_trie_has_prefix(&s_stroke, 1) == false, "S is not prefix");
/* NULL/zero args */
CHECK(steno_trie_lookup(NULL, 1) == NULL, "NULL strokes → NULL");
CHECK(steno_trie_lookup(&s_stroke, 0) == NULL, "0 count → NULL");
CHECK(steno_trie_has_prefix(NULL, 1) == false, "NULL prefix → false");
printf("\n%d/%d tests passed\n", tests_passed, tests_run);
free(data);
return tests_passed == tests_run ? 0 : 1;
}

200
tests/test_undo.c Normal file
View file

@ -0,0 +1,200 @@
/*
* Test undo ring buffer.
* Build: cc -I../src -o test_undo test_undo.c ../src/undo.c
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "undo.h"
static int tests_run = 0;
static int tests_passed = 0;
#define ASSERT(cond, msg) do { \
tests_run++; \
if (!(cond)) { \
printf("FAIL [%s:%d]: %s\n", __func__, __LINE__, msg); \
} else { \
tests_passed++; \
} \
} while (0)
/* 1. Init → count=0, pop returns NULL */
static void test_init(void)
{
struct stroke_history h;
steno_undo_init(&h);
ASSERT(steno_undo_count(&h) == 0, "count should be 0 after init");
ASSERT(steno_undo_pop(&h) == NULL, "pop on empty should return NULL");
ASSERT(steno_undo_peek(&h) == NULL, "peek on empty should return NULL");
}
/* 2. Push one → count=1, peek returns it */
static void test_push_one(void)
{
struct stroke_history h;
steno_undo_init(&h);
uint32_t strokes[] = {0xABCD};
steno_undo_push(&h, strokes, 1, 5, 1, 0x0F);
ASSERT(steno_undo_count(&h) == 1, "count should be 1");
const struct stroke_history_entry *e = steno_undo_peek(&h);
ASSERT(e != NULL, "peek should not be NULL");
ASSERT(e->strokes[0] == 0xABCD, "stroke data mismatch");
ASSERT(e->stroke_count == 1, "stroke_count mismatch");
ASSERT(e->output_len == 5, "output_len mismatch");
ASSERT(e->space_before == 1, "space_before mismatch");
ASSERT(e->fmt_flags == 0x0F, "fmt_flags mismatch");
}
/* 3. Push and pop → entry matches */
static void test_push_pop(void)
{
struct stroke_history h;
steno_undo_init(&h);
uint32_t strokes[] = {0x100, 0x200};
steno_undo_push(&h, strokes, 2, 7, 0, 0x03);
struct stroke_history_entry *e = steno_undo_pop(&h);
ASSERT(e != NULL, "pop should return entry");
ASSERT(e->strokes[0] == 0x100, "stroke[0] mismatch");
ASSERT(e->strokes[1] == 0x200, "stroke[1] mismatch");
ASSERT(e->stroke_count == 2, "stroke_count mismatch");
ASSERT(e->output_len == 7, "output_len mismatch");
ASSERT(e->space_before == 0, "space_before mismatch");
ASSERT(e->fmt_flags == 0x03, "fmt_flags mismatch");
ASSERT(steno_undo_count(&h) == 0, "count should be 0 after pop");
}
/* 4. Overflow: push SIZE+1 → count stays at SIZE */
static void test_overflow(void)
{
struct stroke_history h;
steno_undo_init(&h);
for (int i = 0; i < CONFIG_STENO_HISTORY_SIZE + 1; i++) {
uint32_t s = (uint32_t)i;
steno_undo_push(&h, &s, 1, (uint8_t)(i & 0xFF), 0, 0);
}
ASSERT(steno_undo_count(&h) == CONFIG_STENO_HISTORY_SIZE,
"count should cap at SIZE");
/* Most recent should be SIZE (last pushed) */
const struct stroke_history_entry *e = steno_undo_peek(&h);
ASSERT(e != NULL, "peek should not be NULL");
ASSERT(e->strokes[0] == (uint32_t)CONFIG_STENO_HISTORY_SIZE,
"most recent entry should be last pushed");
/* Oldest (entry 0) should be gone; entry 1 should be oldest */
/* Pop all and check last one is entry 1 */
struct stroke_history_entry *last = NULL;
for (int i = 0; i < CONFIG_STENO_HISTORY_SIZE; i++) {
last = steno_undo_pop(&h);
ASSERT(last != NULL, "pop should succeed");
}
/* last popped = oldest = entry index 1 */
ASSERT(last->strokes[0] == 1, "oldest entry should be index 1 (0 was overwritten)");
ASSERT(steno_undo_count(&h) == 0, "count should be 0 after popping all");
}
/* 5. Pop all → count=0 */
static void test_pop_all(void)
{
struct stroke_history h;
steno_undo_init(&h);
for (int i = 0; i < 10; i++) {
uint32_t s = (uint32_t)i;
steno_undo_push(&h, &s, 1, 1, 0, 0);
}
for (int i = 0; i < 10; i++) {
ASSERT(steno_undo_pop(&h) != NULL, "pop should succeed");
}
ASSERT(steno_undo_count(&h) == 0, "count should be 0");
ASSERT(steno_undo_pop(&h) == NULL, "pop on empty should be NULL");
}
/* 6. Push/pop cycle: push 5, pop 3, push 2, pop 4 → LIFO order */
static void test_push_pop_cycle(void)
{
struct stroke_history h;
steno_undo_init(&h);
/* Push 5: values 10,11,12,13,14 */
for (int i = 0; i < 5; i++) {
uint32_t s = (uint32_t)(10 + i);
steno_undo_push(&h, &s, 1, (uint8_t)(10 + i), 0, 0);
}
ASSERT(steno_undo_count(&h) == 5, "count should be 5");
/* Pop 3: should get 14, 13, 12 */
struct stroke_history_entry *e;
e = steno_undo_pop(&h);
ASSERT(e->strokes[0] == 14, "should pop 14");
e = steno_undo_pop(&h);
ASSERT(e->strokes[0] == 13, "should pop 13");
e = steno_undo_pop(&h);
ASSERT(e->strokes[0] == 12, "should pop 12");
ASSERT(steno_undo_count(&h) == 2, "count should be 2");
/* Push 2: values 20, 21 */
for (int i = 0; i < 2; i++) {
uint32_t s = (uint32_t)(20 + i);
steno_undo_push(&h, &s, 1, (uint8_t)(20 + i), 0, 0);
}
ASSERT(steno_undo_count(&h) == 4, "count should be 4");
/* Pop 4: should get 21, 20, 11, 10 */
e = steno_undo_pop(&h);
ASSERT(e->strokes[0] == 21, "should pop 21");
e = steno_undo_pop(&h);
ASSERT(e->strokes[0] == 20, "should pop 20");
e = steno_undo_pop(&h);
ASSERT(e->strokes[0] == 11, "should pop 11");
e = steno_undo_pop(&h);
ASSERT(e->strokes[0] == 10, "should pop 10");
ASSERT(steno_undo_count(&h) == 0, "count should be 0");
}
/* 7. Verify multi-stroke data preserved */
static void test_data_preservation(void)
{
struct stroke_history h;
steno_undo_init(&h);
uint32_t strokes[] = {0xDEAD, 0xBEEF, 0xCAFE};
steno_undo_push(&h, strokes, 3, 42, 1, 0x7A);
struct stroke_history_entry *e = steno_undo_pop(&h);
ASSERT(e != NULL, "pop should return entry");
ASSERT(e->stroke_count == 3, "stroke_count should be 3");
ASSERT(e->strokes[0] == 0xDEAD, "strokes[0] mismatch");
ASSERT(e->strokes[1] == 0xBEEF, "strokes[1] mismatch");
ASSERT(e->strokes[2] == 0xCAFE, "strokes[2] mismatch");
ASSERT(e->output_len == 42, "output_len mismatch");
ASSERT(e->space_before == 1, "space_before mismatch");
ASSERT(e->fmt_flags == 0x7A, "fmt_flags mismatch");
}
int main(void)
{
printf("Running undo tests...\n\n");
test_init();
test_push_one();
test_push_pop();
test_overflow();
test_pop_all();
test_push_pop_cycle();
test_data_preservation();
printf("\n%d / %d tests passed\n", tests_passed, tests_run);
return (tests_passed == tests_run) ? 0 : 1;
}

553
tools/compile_mphf.py Executable file
View file

@ -0,0 +1,553 @@
#!/usr/bin/env python3
"""CHD MPHF dictionary compiler for steno engine.
Reads Plover JSON dictionaries and produces a compact binary for
embedded use (nRF52840, 462KB flash budget).
"""
import json
import struct
import math
import argparse
import sys
import os
import zlib
from collections import defaultdict
# ─── Steno stroke parsing ───
STENO_KEYS = {
'#': 0x00400000,
'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004,
'P-': 0x00000008, 'W-': 0x00000010, 'H-': 0x00000020,
'R-': 0x00000040, 'A-': 0x00000080, 'O-': 0x00000100,
'*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800,
'-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000,
'-B': 0x00008000, '-L': 0x00010000, '-G': 0x00020000,
'-T': 0x00040000, '-S': 0x00080000, '-D': 0x00100000,
'-Z': 0x00200000,
}
IMPLICIT_HYPHEN = set('AOEU*')
def parse_stroke(s):
result = 0
if '#' in s:
result |= STENO_KEYS['#']
s = s.replace('#', '')
has_hyphen = '-' in s
s_clean = s.replace('-', '')
if not has_hyphen and not any(c in IMPLICIT_HYPHEN for c in s_clean):
for c in s_clean:
key = c + '-'
if key in STENO_KEYS:
result |= STENO_KEYS[key]
return result
if has_hyphen:
hyphen_pos = s.index('-')
for i, c in enumerate(s):
if c == '-':
continue
if c in 'AO':
result |= STENO_KEYS[c + '-']
elif c in 'EU':
result |= STENO_KEYS['-' + c]
elif c == '*':
result |= STENO_KEYS['*']
elif i < hyphen_pos and (c + '-') in STENO_KEYS:
result |= STENO_KEYS[c + '-']
elif i > hyphen_pos and ('-' + c) in STENO_KEYS:
result |= STENO_KEYS['-' + c]
else:
past_vowels = False
for c in s_clean:
if c in 'AO':
result |= STENO_KEYS[c + '-']
past_vowels = True
elif c in 'EU':
result |= STENO_KEYS['-' + c]
past_vowels = True
elif c == '*':
result |= STENO_KEYS['*']
past_vowels = True
elif not past_vowels and (c + '-') in STENO_KEYS:
result |= STENO_KEYS[c + '-']
elif past_vowels and ('-' + c) in STENO_KEYS:
result |= STENO_KEYS['-' + c]
elif (c + '-') in STENO_KEYS:
result |= STENO_KEYS[c + '-']
return result
# ─── Hashing ───
def fnv1a_32(data: bytes) -> int:
"""FNV-1a 32-bit hash."""
h = 0x811c9dc5
for b in data:
h ^= b
h = (h * 0x01000193) & 0xFFFFFFFF
return h
def hash_key(key_bytes: bytes, seed: int) -> int:
"""Hash key with seed by prepending seed bytes."""
return fnv1a_32(struct.pack('<I', seed) + key_bytes)
# ─── Bit packing ───
class BitWriter:
def __init__(self):
self.data = bytearray()
self.bit_pos = 0 # total bits written
def write_bits(self, value, n_bits):
"""Write n_bits of value (LSB first)."""
for i in range(n_bits):
if self.bit_pos % 8 == 0:
self.data.append(0)
if value & (1 << i):
self.data[-1] |= (1 << (self.bit_pos % 8))
self.bit_pos += 1
def pad_to_alignment(self, alignment=4):
"""Pad to byte alignment."""
while len(self.data) % alignment != 0:
self.data.append(0)
self.bit_pos = len(self.data) * 8
def to_bytes(self):
return bytes(self.data)
class BitReader:
def __init__(self, data):
self.data = data
self.bit_pos = 0
def read_bits(self, n_bits):
value = 0
for i in range(n_bits):
byte_idx = self.bit_pos // 8
bit_idx = self.bit_pos % 8
if self.data[byte_idx] & (1 << bit_idx):
value |= (1 << i)
self.bit_pos += 1
return value
# ─── Key encoding ───
def encode_key(stroke_str):
"""Parse stroke string → key_bytes (each stroke as u32 LE, concatenated)."""
parts = stroke_str.split('/')
strokes = tuple(parse_stroke(s) for s in parts)
key_bytes = b''.join(struct.pack('<I', s) for s in strokes)
return strokes, key_bytes
# ─── Importance scoring ───
def score_entry(stroke_str, translation):
"""Lower score = more important = keep first."""
n_strokes = stroke_str.count('/') + 1
has_format = '{' in translation
return (n_strokes, has_format, len(translation), stroke_str)
# ─── CHD MPHF construction ───
def build_chd(keys_and_bytes, entry_count):
"""
Build CHD MPHF.
keys_and_bytes: list of (index, key_bytes) for each entry
entry_count: total number of entries
Returns: (displacements, slot_to_entry_idx, max_displacement)
displacements[bucket] = d value
slot_to_entry_idx[slot] = index into keys_and_bytes, or -1 if empty
"""
bucket_count = max(entry_count // 3, min(entry_count, 16))
# Assign keys to buckets
buckets = defaultdict(list)
for idx, (_, kb) in enumerate(keys_and_bytes):
b = hash_key(kb, 0) % bucket_count
buckets[b].append(idx)
# Sort buckets by size descending
sorted_buckets = sorted(buckets.items(), key=lambda x: len(x[1]), reverse=True)
displacements = [0] * bucket_count
occupied = set()
slot_to_entry = [-1] * entry_count
max_disp = 0
for bucket_id, members in sorted_buckets:
if not members:
continue
member_key_bytes = [(m, keys_and_bytes[m][1]) for m in members]
placed = False
for d in range(65536):
slots = []
collision = False
seen = set()
for _, kb in member_key_bytes:
slot = hash_key(kb, d + 1) % entry_count
if slot in occupied or slot in seen:
collision = True
break
seen.add(slot)
slots.append(slot)
if collision:
continue
# Place all members
for i, (m, _) in enumerate(member_key_bytes):
occupied.add(slots[i])
slot_to_entry[slots[i]] = m
displacements[bucket_id] = d
if d > max_disp:
max_disp = d
placed = True
break
if not placed:
print(f"FATAL: bucket {bucket_id} with {len(members)} keys failed after 65536 tries",
file=sys.stderr)
return None, None, None
return displacements, slot_to_entry, max_disp
# ─── Compilation ───
def compile_mphf(entries, max_size=None):
"""
entries: list of (stroke_str, translation) from JSON dict
max_size: max output size in bytes (default: 462*1024 = 473088)
Returns: bytes (the compiled binary) or None if can't fit
"""
if max_size is None:
max_size = 462 * 1024
# Sort by importance for potential trimming
entries_scored = sorted(entries, key=lambda e: score_entry(e[0], e[1]))
# Parse all keys, dedup by key_bytes (last wins for same key)
seen_keys = {}
for stroke_str, translation in entries_scored:
strokes, key_bytes = encode_key(stroke_str)
if key_bytes in seen_keys:
prev = seen_keys[key_bytes]
print(f" Dedup: '{stroke_str}''{translation}' collides with "
f"'{prev[2]}''{prev[1]}', keeping first", file=sys.stderr)
continue
entry = (key_bytes, translation, stroke_str, strokes)
seen_keys[key_bytes] = entry
keys_and_bytes = list(seen_keys.values())
# Iteratively trim if needed
while True:
entry_count = len(keys_and_bytes)
if entry_count == 0:
return None
# Build deduped string table
translations = [kb[1] for kb in keys_and_bytes]
unique_translations = sorted(set(translations))
trans_to_id = {t: i for i, t in enumerate(unique_translations)}
unique_count = len(unique_translations)
# Estimate size
bucket_count = max(entry_count // 3, min(entry_count, 16))
est_value_bits = max(1, math.ceil(math.log2(max(unique_count, 2))))
est_disp_bits = 16 # conservative
est_disp_bytes = (bucket_count * est_disp_bits + 7) // 8
est_disp_bytes = ((est_disp_bytes + 3) // 4) * 4
est_value_bytes = (entry_count * est_value_bits + 7) // 8
est_value_bytes = ((est_value_bytes + 3) // 4) * 4
est_fp_bytes = ((entry_count + 3) // 4) * 4
# String table (block-compressed)
string_data_raw = b''
string_offsets = []
for t in unique_translations:
string_offsets.append(len(string_data_raw))
string_data_raw += t.encode('utf-8') + b'\x00'
block_size = 4096
compressed_blocks = []
for i in range(0, len(string_data_raw), block_size):
block = string_data_raw[i:i + block_size]
compressed_blocks.append(zlib.compress(block, 9))
est_str_offsets = unique_count * 3 # u24 packed LE
est_block_dir = 2 + len(compressed_blocks) * 4 # u16 count + u32 offsets
est_str_data = sum(len(b) for b in compressed_blocks) + est_block_dir
# Prefix table
prefix_strokes = set()
for kb, trans, stroke_str, strokes in keys_and_bytes:
if len(strokes) > 1:
prefix_strokes.add(strokes[0])
prefix_list = sorted(prefix_strokes)
est_prefix = len(prefix_list) * 4
total_est = 32 + est_disp_bytes + est_value_bytes + est_fp_bytes + est_str_offsets + est_str_data + est_prefix
if total_est <= max_size:
break
# Trim: use ratio of overshoot to estimate how many entries to cut
overshoot_ratio = total_est / max_size
target_entries = int(entry_count / overshoot_ratio * 0.98) # 2% safety margin
trim_count = max(1, entry_count - target_entries)
keys_and_bytes = keys_and_bytes[:entry_count - trim_count]
print(f" Trimming to {len(keys_and_bytes)} entries (est {total_est} > {max_size})",
file=sys.stderr)
entry_count = len(keys_and_bytes)
bucket_count = max(entry_count // 3, min(entry_count, 16))
print(f" Building CHD MPHF: {entry_count} entries, {bucket_count} buckets...",
file=sys.stderr)
# Build CHD
chd_input = [(i, keys_and_bytes[i][0]) for i in range(entry_count)]
displacements, slot_to_entry, max_disp = build_chd(chd_input, entry_count)
if displacements is None:
return None
# Compute actual bit widths
disp_bits = max(1, math.ceil(math.log2(max(max_disp + 1, 2))))
value_bits = max(1, math.ceil(math.log2(max(unique_count, 2))))
prefix_count = len(prefix_list)
print(f" Max displacement: {max_disp}, disp_bits: {disp_bits}", file=sys.stderr)
print(f" Unique translations: {unique_count}, value_bits: {value_bits}", file=sys.stderr)
print(f" Prefix entries: {prefix_count}", file=sys.stderr)
# ─── Build binary ───
# Displacements section
disp_writer = BitWriter()
for d in displacements:
disp_writer.write_bits(d, disp_bits)
disp_writer.pad_to_alignment(4)
disp_section = disp_writer.to_bytes()
# Values section: slot → value_id
val_writer = BitWriter()
fingerprints = bytearray(entry_count)
for slot in range(entry_count):
entry_idx = slot_to_entry[slot]
if entry_idx >= 0:
kb, trans, stroke_str, strokes = keys_and_bytes[entry_idx]
val_id = trans_to_id[trans]
val_writer.write_bits(val_id, value_bits)
fingerprints[slot] = fnv1a_32(kb) & 0xFF
else:
val_writer.write_bits(0, value_bits)
fingerprints[slot] = 0
val_writer.pad_to_alignment(4)
val_section = val_writer.to_bytes()
# Fingerprints section
fp_section = bytes(fingerprints)
# Pad to 4-byte boundary
while len(fp_section) % 4 != 0:
fp_section += b'\x00'
# String offsets section (u24 packed LE — 3 bytes each, into raw/uncompressed table)
str_offsets_section = b''.join(struct.pack('<I', off)[:3] for off in string_offsets)
# String data section (block-compressed)
block_dir = struct.pack('<H', len(compressed_blocks))
block_offset = 0
for blk in compressed_blocks:
block_dir += struct.pack('<I', block_offset)
block_offset += len(blk)
str_data_section = block_dir + b''.join(compressed_blocks)
# Prefix table section
prefix_section = b''.join(struct.pack('<I', s) for s in prefix_list)
# Header (32 bytes):
# magic: u32, version: u16, flags: u16,
# entry_count: u32, bucket_count: u32, unique_count: u32,
# value_bits: u8, disp_bits: u8, prefix_count: u16,
# reserved0: u32, reserved1: u32
header = struct.pack('<IHHIIIBBHii',
0x4F4E5453, # magic "STNO"
2, # version
0x0001, # flags: bit 0 = block-compressed strings
entry_count, # entry_count
bucket_count, # bucket_count
unique_count, # unique_count
value_bits, # value_bits
disp_bits, # disp_bits
prefix_count, # prefix_count
0, # reserved0
0, # reserved1
)
assert len(header) == 32, f"Header is {len(header)} bytes, expected 32"
binary = header + disp_section + val_section + fp_section + str_offsets_section + str_data_section + prefix_section
# ─── Verification ───
print(f" Verifying all {entry_count} entries...", file=sys.stderr)
errors = 0
for entry_idx in range(entry_count):
kb, trans, stroke_str, strokes = keys_and_bytes[entry_idx]
# Lookup through MPHF
bucket = hash_key(kb, 0) % bucket_count
# Read displacement
disp_reader = BitReader(disp_section)
disp_reader.bit_pos = bucket * disp_bits
d = disp_reader.read_bits(disp_bits)
slot = hash_key(kb, d + 1) % entry_count
# Check fingerprint
expected_fp = fnv1a_32(kb) & 0xFF
if fingerprints[slot] != expected_fp:
print(f" VERIFY FAIL: fingerprint mismatch for '{stroke_str}' at slot {slot}: "
f"got {fingerprints[slot]}, expected {expected_fp}", file=sys.stderr)
errors += 1
continue
# Check value
val_reader = BitReader(val_section)
val_reader.bit_pos = slot * value_bits
val_id = val_reader.read_bits(value_bits)
# Resolve string from compressed table
off_bytes = str_offsets_section[val_id * 3:(val_id + 1) * 3]
str_off = off_bytes[0] | (off_bytes[1] << 8) | (off_bytes[2] << 16)
block_idx = str_off // block_size
in_block_off = str_off % block_size
raw_block = zlib.decompress(compressed_blocks[block_idx])
if b'\x00' in raw_block[in_block_off:]:
end = raw_block.index(b'\x00', in_block_off)
resolved = raw_block[in_block_off:end].decode('utf-8')
elif block_idx + 1 < len(compressed_blocks):
part1 = raw_block[in_block_off:]
next_block = zlib.decompress(compressed_blocks[block_idx + 1])
end = next_block.index(b'\x00')
resolved = (part1 + next_block[:end]).decode('utf-8')
else:
resolved = raw_block[in_block_off:].decode('utf-8')
if resolved != trans:
print(f" VERIFY FAIL: value mismatch for '{stroke_str}': "
f"got '{resolved}', expected '{trans}'", file=sys.stderr)
errors += 1
if errors:
print(f" VERIFICATION FAILED: {errors} errors", file=sys.stderr)
return None
print(f" Verification passed: all {entry_count} entries OK", file=sys.stderr)
# Check final size
if len(binary) > max_size:
print(f" WARNING: output {len(binary)} bytes exceeds max {max_size}", file=sys.stderr)
return binary, {
'entry_count': entry_count,
'bucket_count': bucket_count,
'unique_count': unique_count,
'value_bits': value_bits,
'disp_bits': disp_bits,
'max_displacement': max_disp,
'prefix_count': prefix_count,
'disp_section_bytes': len(disp_section),
'val_section_bytes': len(val_section),
'fp_section_bytes': len(fp_section),
'str_offsets_bytes': len(str_offsets_section),
'str_data_bytes': len(str_data_section),
'str_data_raw_bytes': len(string_data_raw),
'prefix_section_bytes': len(prefix_section),
'total_bytes': len(binary),
}
def print_stats(stats):
"""Print size breakdown statistics."""
total = stats['entry_count']
print(f"Entries: {stats['entry_count']}")
print(f"MPHF displacements: {stats['disp_section_bytes']/1024:.1f} KB "
f"({stats['bucket_count']} buckets, {stats['disp_bits']} bits each)")
print(f"Value array: {stats['val_section_bytes']/1024:.1f} KB "
f"({stats['entry_count']} entries, {stats['value_bits']} bits each)")
print(f"Fingerprints: {stats['fp_section_bytes']/1024:.1f} KB")
print(f"String offsets: {stats['str_offsets_bytes']/1024:.1f} KB "
f"({stats['unique_count']} unique x 3 bytes)")
print(f"String data: {stats['str_data_bytes']/1024:.1f} KB"
f" (compressed, {stats.get('str_data_raw_bytes', 0)/1024:.1f} KB raw)")
print(f"Prefix table: {stats['prefix_section_bytes']/1024:.1f} KB "
f"({stats['prefix_count']} entries x 4 bytes)")
print(f"Total: {stats['total_bytes']/1024:.1f} KB")
def main():
parser = argparse.ArgumentParser(description='Compile steno dictionary to MPHF binary format')
parser.add_argument('input', help='Input JSON dictionary (Plover format)')
parser.add_argument('output', help='Output binary file')
parser.add_argument('--max-size', type=int, default=462*1024,
help='Maximum output size in bytes (default: 473088 = 462KB)')
parser.add_argument('--max-entries', type=int, default=None,
help='Maximum number of entries')
parser.add_argument('--stats', action='store_true',
help='Print size breakdown statistics')
parser.add_argument('--verify', action='store_true', default=True,
help='Verify compiled dict (default: true)')
args = parser.parse_args()
# Load dictionary
with open(args.input) as f:
raw_dict = json.load(f)
print(f"Loaded {len(raw_dict)} entries from {args.input}", file=sys.stderr)
entries = list(raw_dict.items())
if args.max_entries is not None:
entries_scored = sorted(entries, key=lambda e: score_entry(e[0], e[1]))
entries = entries_scored[:args.max_entries]
print(f"Trimmed to {len(entries)} entries (--max-entries)", file=sys.stderr)
result = compile_mphf(entries, max_size=args.max_size)
if result is None:
print("Compilation failed", file=sys.stderr)
sys.exit(1)
binary, stats = result
with open(args.output, 'wb') as f:
f.write(binary)
print(f"Wrote {len(binary)} bytes to {args.output}", file=sys.stderr)
if args.stats:
print()
print_stats(stats)
if __name__ == '__main__':
main()

184
tools/compile_simple.py Normal file
View file

@ -0,0 +1,184 @@
#!/usr/bin/env python3
"""Simple flat-format steno dictionary compiler.
Outputs a binary format optimized for binary search on embedded targets.
No compression just sorted entries with fixed-width keys.
Format:
Header (16 bytes):
magic: u32 = 0x4F4E5453 ("STNO")
version: u16 = 1
max_strokes: u8 (max stroke count per entry)
pad: u8
entry_count: u32
strings_offset: u32
Entry array (sorted by stroke tuple, fixed width):
Each entry = max_strokes * 4 + 4 bytes:
strokes[max_strokes]: u32 LE (unused slots = 0)
string_offset: u32 LE (into string table)
String table:
Null-terminated UTF-8 strings, concatenated
"""
import argparse
import json
import struct
import sys
import os
STENO_KEYS = {
'#': 0x00400000,
'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004,
'P-': 0x00000008, 'W-': 0x00000010, 'H-': 0x00000020,
'R-': 0x00000040, 'A-': 0x00000080, 'O-': 0x00000100,
'*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800,
'-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000,
'-B': 0x00008000, '-L': 0x00010000, '-G': 0x00020000,
'-T': 0x00040000, '-S': 0x00080000, '-D': 0x00100000,
'-Z': 0x00200000,
}
IMPLICIT_HYPHEN = set('AOEU*')
def parse_stroke(s):
result = 0
if '#' in s:
result |= STENO_KEYS['#']
s = s.replace('#', '')
has_hyphen = '-' in s
s_clean = s.replace('-', '')
if not has_hyphen and not any(c in IMPLICIT_HYPHEN for c in s_clean):
for c in s_clean:
key = c + '-'
if key in STENO_KEYS:
result |= STENO_KEYS[key]
return result
if has_hyphen:
hyphen_pos = s.index('-')
for i, c in enumerate(s):
if c == '-':
continue
if c in 'AO':
result |= STENO_KEYS[c + '-']
elif c in 'EU':
result |= STENO_KEYS['-' + c]
elif c == '*':
result |= STENO_KEYS['*']
elif i < hyphen_pos and (c + '-') in STENO_KEYS:
result |= STENO_KEYS[c + '-']
elif i > hyphen_pos and ('-' + c) in STENO_KEYS:
result |= STENO_KEYS['-' + c]
else:
past_vowels = False
for c in s_clean:
if c in 'AO':
result |= STENO_KEYS[c + '-']
past_vowels = True
elif c in 'EU':
result |= STENO_KEYS['-' + c]
past_vowels = True
elif c == '*':
result |= STENO_KEYS['*']
past_vowels = True
elif not past_vowels and (c + '-') in STENO_KEYS:
result |= STENO_KEYS[c + '-']
elif past_vowels and ('-' + c) in STENO_KEYS:
result |= STENO_KEYS['-' + c]
elif (c + '-') in STENO_KEYS:
result |= STENO_KEYS[c + '-']
return result
def compile_dict(json_path, max_entries=None, max_strokes_cap=None):
with open(json_path) as f:
raw = json.load(f)
entries = []
for stroke_str, translation in raw.items():
strokes = tuple(parse_stroke(s) for s in stroke_str.split('/'))
if max_strokes_cap and len(strokes) > max_strokes_cap:
continue
entries.append((strokes, translation))
if max_entries and len(entries) > max_entries:
single = [(s, t) for s, t in entries if len(s) == 1]
multi = [(s, t) for s, t in entries if len(s) > 1]
multi.sort(key=lambda x: (len(x[1]), len(x[0])))
remaining = max_entries - len(single)
if remaining < 0:
single.sort(key=lambda x: len(x[1]))
entries = single[:max_entries]
else:
entries = single + multi[:remaining]
entries.sort(key=lambda x: x[0])
max_strokes = max(len(s) for s, _ in entries)
string_table = bytearray()
string_offsets = {}
for _, translation in entries:
if translation not in string_offsets:
string_offsets[translation] = len(string_table)
string_table.extend(translation.encode('utf-8'))
string_table.append(0)
entry_size = max_strokes * 4 + 4
header_size = 16
entries_size = len(entries) * entry_size
strings_offset = header_size + entries_size
header = struct.pack('<IHBBII',
0x4F4E5453, # "STNO"
1, # version
max_strokes,
0, # pad
len(entries),
strings_offset)
entry_data = bytearray()
for strokes, translation in entries:
padded = list(strokes) + [0] * (max_strokes - len(strokes))
for s in padded:
entry_data.extend(struct.pack('<I', s))
entry_data.extend(struct.pack('<I', string_offsets[translation]))
binary = header + bytes(entry_data) + bytes(string_table)
return binary, len(entries), max_strokes, len(string_table)
def main():
parser = argparse.ArgumentParser(description='Compile steno dict to flat binary')
parser.add_argument('input', help='JSON dictionary path')
parser.add_argument('-o', '--output', default='steno_dict.bin')
parser.add_argument('--max-entries', type=int, default=None)
parser.add_argument('--max-strokes', type=int, default=None,
help='Cap max stroke count per entry (drop longer)')
parser.add_argument('--stats', action='store_true')
args = parser.parse_args()
if not os.path.exists(args.input):
print(f"ERROR: {args.input} not found")
sys.exit(1)
binary, n_entries, max_strokes, str_size = compile_dict(
args.input, args.max_entries, args.max_strokes)
with open(args.output, 'wb') as f:
f.write(binary)
if args.stats:
entry_size = max_strokes * 4 + 4
print(f"Entries: {n_entries}")
print(f"Max strokes: {max_strokes}")
print(f"Entry size: {entry_size} bytes")
print(f"Entry array: {n_entries * entry_size} bytes")
print(f"String table: {str_size} bytes")
print(f"Total: {len(binary)} bytes ({len(binary)/1024:.1f} KB)")
print(f"Written {len(binary)} bytes to {args.output}")
if __name__ == '__main__':
main()

815
tools/dict_compiler.py Normal file
View file

@ -0,0 +1,815 @@
#!/usr/bin/env python3
"""DAWG dictionary compiler for ZMK steno engine.
Compiles a Plover-format JSON steno dictionary into a compact binary
DAWG with skip-count indexing and block-compressed string table.
Optimize-dict variant targeting 462KB (left half flash budget).
"""
import argparse
import json
import math
import os
import struct
import sys
import zlib
from collections import Counter
# ─── Steno stroke parsing (copied from dawg_fst_prototype.py) ───
STENO_KEYS = {
'#': 0x00400000,
'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004,
'P-': 0x00000008, 'W-': 0x00000010, 'H-': 0x00000020,
'R-': 0x00000040, 'A-': 0x00000080, 'O-': 0x00000100,
'*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800,
'-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000,
'-B': 0x00008000, '-L': 0x00010000, '-G': 0x00020000,
'-T': 0x00040000, '-S': 0x00080000, '-D': 0x00100000,
'-Z': 0x00200000,
}
IMPLICIT_HYPHEN = set('AOEU*')
def parse_stroke(s):
"""Parse a steno stroke string into a bitmask."""
result = 0
if '#' in s:
result |= STENO_KEYS['#']
s = s.replace('#', '')
has_hyphen = '-' in s
s_clean = s.replace('-', '')
if not has_hyphen and not any(c in IMPLICIT_HYPHEN for c in s_clean):
for c in s_clean:
key = c + '-'
if key in STENO_KEYS:
result |= STENO_KEYS[key]
return result
past_vowels = False
for c in s_clean:
if c in 'AO':
result |= STENO_KEYS[c + '-']
past_vowels = True
elif c in 'EU':
result |= STENO_KEYS['-' + c]
past_vowels = True
elif c == '*':
result |= STENO_KEYS['*']
past_vowels = True
elif not past_vowels and (c + '-') in STENO_KEYS:
result |= STENO_KEYS[c + '-']
elif past_vowels and ('-' + c) in STENO_KEYS:
result |= STENO_KEYS['-' + c]
elif has_hyphen:
if s.index(c) < s.index('-'):
result |= STENO_KEYS.get(c + '-', 0)
else:
result |= STENO_KEYS.get('-' + c, 0)
else:
if (c + '-') in STENO_KEYS:
result |= STENO_KEYS[c + '-']
return result
def parse_stroke_string(stroke_str):
"""Parse a stroke string (possibly multi-stroke with /) into tuple of bitmasks."""
return tuple(parse_stroke(s) for s in stroke_str.split('/'))
# ─── DAWG construction (Daciuk's incremental algorithm) ───
class DawgNode:
"""Node in the DAWG."""
__slots__ = ['id', 'edges', 'final', '_hash_cache']
_next_id = 0
def __init__(self):
self.id = DawgNode._next_id
DawgNode._next_id += 1
self.edges = {} # stroke_val -> DawgNode
self.final = False
self._hash_cache = None
def signature(self):
"""Hashable signature for minimization."""
edge_sig = tuple(sorted(
(k, child.id) for k, child in self.edges.items()
))
return (self.final, edge_sig)
def __hash__(self):
if self._hash_cache is None:
self._hash_cache = hash(self.signature())
return self._hash_cache
def __eq__(self, other):
return self.signature() == other.signature()
def invalidate_cache(self):
self._hash_cache = None
def build_dawg(sorted_entries):
"""Build minimized DAWG using Daciuk's incremental algorithm.
Entries MUST be sorted by stroke tuple (lexicographic).
Returns (root, node_count, edge_count).
"""
DawgNode._next_id = 0
root = DawgNode()
unchecked = [] # list of (parent, stroke, child)
minimized = {} # signature -> node
prev_strokes = ()
def _minimize(down_to):
"""Minimize unchecked nodes from top down to given depth."""
for i in range(len(unchecked) - 1, down_to - 1, -1):
parent, stroke, child = unchecked[i]
child.invalidate_cache()
sig = child.signature()
if sig in minimized:
parent.edges[stroke] = minimized[sig]
else:
minimized[sig] = child
unchecked.pop()
for strokes, _translation in sorted_entries:
# Find common prefix length with previous entry
common = 0
limit = min(len(strokes), len(prev_strokes))
while common < limit and strokes[common] == prev_strokes[common]:
common += 1
# Minimize nodes beyond common prefix
_minimize(common)
# Get node at end of common prefix
if unchecked:
node = unchecked[-1][2]
else:
node = root
# Add new nodes for remaining strokes
for stroke in strokes[common:]:
new_node = DawgNode()
node.edges[stroke] = new_node
unchecked.append((node, stroke, new_node))
node = new_node
node.final = True
prev_strokes = strokes
# Minimize remaining
_minimize(0)
# Count nodes and edges
node_count = 0
edge_count = 0
visited = set()
def _count(n):
nonlocal node_count, edge_count
if n.id in visited:
return
visited.add(n.id)
node_count += 1
for _s, child in n.edges.items():
edge_count += 1
_count(child)
_count(root)
return root, node_count, edge_count
# ─── Skip-count computation ───
def compute_skip_counts(root):
"""Compute skip-count (number of final nodes in subtree) for each node.
Returns dict: node_id -> skip_count
"""
cache = {}
def _count(node):
if node.id in cache:
return cache[node.id]
c = 1 if node.final else 0
for stroke in sorted(node.edges.keys()):
child = node.edges[stroke]
c += _count(child)
cache[node.id] = c
return c
_count(root)
return cache
def dawg_lookup_index(root, strokes, skip_cache):
"""Look up a stroke sequence in the DAWG, returning its skip-count index.
Returns -1 if not found.
"""
node = root
idx = 0
for stroke in strokes:
if stroke not in node.edges:
return -1
# Count finals of all edges with stroke < target
for s in sorted(node.edges.keys()):
if s == stroke:
child = node.edges[s]
if child.final:
idx += 1
node = child
break
else:
child = node.edges[s]
idx += skip_cache[child.id]
else:
return -1
if not node.final:
return -1
return idx - 1
def get_dawg_traversal_order(root):
"""DFS traversal of DAWG, edges sorted by stroke value.
Returns list of translations in traversal order (one per final node encounter).
This is the order entries appear when looking up via skip-count.
"""
order = []
visited_paths = set()
def _dfs(node, path):
path_key = tuple(path)
if path_key in visited_paths:
return
visited_paths.add(path_key)
if node.final:
order.append(path_key)
for stroke in sorted(node.edges.keys()):
child = node.edges[stroke]
_dfs(child, path + [stroke])
_dfs(root, [])
return order
# ─── Entry trimming ───
def trim_entries(entries, max_entries):
"""Trim entries to max_entries, keeping single-stroke preferentially.
Priority:
1. All single-stroke entries
2. Multi-stroke entries sorted by translation length (shorter first)
3. Drop longest/rarest multi-stroke first
"""
if len(entries) <= max_entries:
return entries
single_stroke = []
multi_stroke = []
for strokes_str, translation in entries:
if '/' not in strokes_str:
single_stroke.append((strokes_str, translation))
else:
multi_stroke.append((strokes_str, translation))
# Sort multi-stroke by translation length (shorter = more useful)
multi_stroke.sort(key=lambda x: (len(x[1]), len(x[0].split('/'))))
remaining = max_entries - len(single_stroke)
if remaining < 0:
# Even single-stroke entries exceed limit; trim by translation length
single_stroke.sort(key=lambda x: len(x[1]))
return single_stroke[:max_entries]
return single_stroke + multi_stroke[:remaining]
# ─── String table construction ───
def build_string_table(translations):
"""Build block-compressed string table from translations.
Returns (table_bytes, offsets) where offsets[i] is the byte offset
of translation i in the uncompressed table.
"""
# Build raw table: null-separated strings
# We need to track offset of each unique string
unique_translations = sorted(set(translations))
trans_to_unique_idx = {t: i for i, t in enumerate(unique_translations)}
# Build raw bytes and offset map
raw_parts = []
unique_offsets = []
offset = 0
for t in unique_translations:
unique_offsets.append(offset)
encoded = t.encode('utf-8')
raw_parts.append(encoded)
offset += len(encoded) + 1 # +1 for null separator
raw = b'\x00'.join(raw_parts)
if raw_parts:
raw += b'\x00' # trailing null
# Block compress
block_size = 4096
compressed_blocks = []
block_offsets_raw = []
current_offset = 0
for i in range(0, len(raw), block_size):
block = raw[i:i + block_size]
compressed = zlib.compress(block, 9)
block_offsets_raw.append(current_offset)
compressed_blocks.append(compressed)
current_offset += len(compressed)
# Serialize: block_count(u16) + block_offsets(u32 each) + compressed blocks
n_blocks = len(compressed_blocks)
table_header = struct.pack('<H', n_blocks)
table_index = b''.join(struct.pack('<I', off) for off in block_offsets_raw)
table_data = b''.join(compressed_blocks)
table_bytes = table_header + table_index + table_data
# Map each translation to its offset in raw table
entry_offsets = []
for t in translations:
uid = trans_to_unique_idx[t]
entry_offsets.append(unique_offsets[uid])
return table_bytes, entry_offsets, len(raw)
def decompress_string_table(table_bytes):
"""Decompress a block-compressed string table back to raw bytes."""
pos = 0
n_blocks = struct.unpack_from('<H', table_bytes, pos)[0]
pos += 2
block_offsets = []
for _ in range(n_blocks):
off = struct.unpack_from('<I', table_bytes, pos)[0]
pos += 4
block_offsets.append(off)
data_start = pos
raw_parts = []
for i in range(n_blocks):
block_start = data_start + block_offsets[i]
if i + 1 < n_blocks:
block_end = data_start + block_offsets[i + 1]
else:
block_end = len(table_bytes)
compressed = table_bytes[block_start:block_end]
raw_parts.append(zlib.decompress(compressed))
return b''.join(raw_parts)
def lookup_string(raw_table, offset):
"""Look up a null-terminated string at given offset in raw table."""
end = raw_table.index(b'\x00', offset)
return raw_table[offset:end].decode('utf-8')
# ─── Binary serialization ───
MAGIC = b'STNO'
VERSION = 1
FLAG_SPLIT_STORAGE = 0x0001
HEADER_SIZE = 32
def serialize_header(flags, entry_count, node_count, edge_count,
string_table_offset, string_table_size,
value_array_offset):
"""Serialize the 32-byte binary header."""
return struct.pack('<4sHHIIIIIIxxxx',
MAGIC,
VERSION,
flags,
entry_count,
node_count,
edge_count,
string_table_offset,
string_table_size,
value_array_offset)
def parse_header(data):
"""Parse 32-byte binary header. Returns dict."""
if len(data) < HEADER_SIZE:
raise ValueError("Data too short for header")
# Unpack with padding bytes
magic, version, flags, entry_count, node_count, edge_count, \
str_table_off, str_table_size, val_array_off = \
struct.unpack_from('<4sHHIIIIII', data, 0)
# 4 bytes reserved at end (32 - 28 = 4)
if magic != MAGIC:
raise ValueError(f"Bad magic: {magic!r}")
return {
'magic': magic,
'version': version,
'flags': flags,
'entry_count': entry_count,
'node_count': node_count,
'edge_count': edge_count,
'string_table_offset': str_table_off,
'string_table_size': str_table_size,
'value_array_offset': val_array_off,
}
def serialize_edges(root, node_count, edge_count):
"""Serialize DAWG edges as bit-packed array.
Each edge: stroke_key(16) + target_node(16) + skip_count(17) + is_last(1) = 50 bits
Nodes are assigned sequential IDs via DFS traversal (sorted edges).
Returns (edge_bytes, node_id_map, skip_counts_by_node).
"""
# Assign sequential node IDs via DFS
node_id_map = {}
dfs_order = []
def _assign_ids(node):
if node.id in node_id_map:
return
new_id = len(node_id_map)
node_id_map[node.id] = new_id
dfs_order.append(node)
for stroke in sorted(node.edges.keys()):
child = node.edges[stroke]
_assign_ids(child)
_assign_ids(root)
# Compute skip counts
skip_cache = compute_skip_counts(root)
# Build edge list: for each node in DFS order, emit edges sorted by stroke
edges = []
for node in dfs_order:
sorted_strokes = sorted(node.edges.keys())
for i, stroke in enumerate(sorted_strokes):
child = node.edges[stroke]
is_last = (i == len(sorted_strokes) - 1)
target_id = node_id_map[child.id]
skip = skip_cache[child.id]
edges.append((stroke, target_id, skip, is_last))
# Bit-pack edges: each 50 bits
# stroke_key: 16 bits, target_node: 16 bits, skip_count: 17 bits, is_last: 1 bit
total_bits = len(edges) * 50
total_bytes = (total_bits + 7) // 8
buf = bytearray(total_bytes)
bit_pos = 0
for stroke_key, target_node, skip_count, is_last in edges:
# Clamp values to field widths
stroke_key &= 0xFFFF
target_node &= 0xFFFF
skip_count = min(skip_count, 0x1FFFF) # 17 bits max
is_last_bit = 1 if is_last else 0
# Pack 50 bits: stroke(16) | target(16) | skip(17) | last(1)
val = (stroke_key << 34) | (target_node << 18) | (skip_count << 1) | is_last_bit
# Write 50 bits into buffer at bit_pos
for i in range(50):
bit = (val >> (49 - i)) & 1
byte_idx = (bit_pos + i) // 8
bit_idx = 7 - ((bit_pos + i) % 8)
if bit:
buf[byte_idx] |= (1 << bit_idx)
bit_pos += 50
return bytes(buf), node_id_map, skip_cache
def deserialize_edges(edge_bytes, edge_count):
"""Deserialize bit-packed edge array.
Returns list of (stroke_key, target_node, skip_count, is_last).
"""
edges = []
bit_pos = 0
for _ in range(edge_count):
val = 0
for i in range(50):
byte_idx = (bit_pos + i) // 8
bit_idx = 7 - ((bit_pos + i) % 8)
bit = (edge_bytes[byte_idx] >> bit_idx) & 1
val = (val << 1) | bit
bit_pos += 50
stroke_key = (val >> 34) & 0xFFFF
target_node = (val >> 18) & 0xFFFF
skip_count = (val >> 1) & 0x1FFFF
is_last = val & 1
edges.append((stroke_key, target_node, skip_count, bool(is_last)))
return edges
def serialize_value_array(entry_offsets, raw_table_size):
"""Serialize value array (string table offsets for each entry).
Uses uint16 if raw_table_size <= 65535, else uint32.
"""
use_u32 = raw_table_size > 65535
fmt = '<I' if use_u32 else '<H'
parts = [struct.pack(fmt, off) for off in entry_offsets]
return b''.join(parts), use_u32
def deserialize_value_array(data, entry_count, use_u32=False):
"""Deserialize value array."""
fmt = '<I' if use_u32 else '<H'
size = 4 if use_u32 else 2
offsets = []
for i in range(entry_count):
off = struct.unpack_from(fmt, data, i * size)[0]
offsets.append(off)
return offsets
# ─── Full compilation pipeline ───
def compile_dictionary(json_path, max_entries=120000, split_storage=False):
"""Compile a JSON steno dictionary into binary DAWG format.
Returns (binary_data, stats_dict).
"""
# 1. Load and parse
with open(json_path) as f:
raw_dict = json.load(f)
raw_entries = list(raw_dict.items())
# 2. Trim if needed
if len(raw_entries) > max_entries:
raw_entries = trim_entries(raw_entries, max_entries)
# 3. Parse strokes and sort
parsed = []
for stroke_str, translation in raw_entries:
strokes = parse_stroke_string(stroke_str)
parsed.append((strokes, translation))
parsed.sort(key=lambda x: x[0])
# 4. Build DAWG
root, node_count, edge_count = build_dawg(parsed)
# 5. Get traversal order for value array
traversal_paths = get_dawg_traversal_order(root)
# Build path->translation map
path_to_trans = {}
for strokes, translation in parsed:
path_to_trans[strokes] = translation
translations_ordered = []
for path in traversal_paths:
if path in path_to_trans:
translations_ordered.append(path_to_trans[path])
else:
translations_ordered.append("")
# 6. Build string table
string_table_bytes, entry_offsets, raw_table_size = \
build_string_table(translations_ordered)
# 7. Serialize edges
edge_bytes, node_id_map, skip_cache = \
serialize_edges(root, node_count, edge_count)
# 8. Serialize value array
value_array_bytes, use_u32 = \
serialize_value_array(entry_offsets, raw_table_size)
# 9. Compute offsets
edge_array_offset = HEADER_SIZE
value_array_offset = edge_array_offset + len(edge_bytes)
string_table_offset = value_array_offset + len(value_array_bytes)
# 10. Build header
flags = 0
if split_storage:
flags |= FLAG_SPLIT_STORAGE
if use_u32:
flags |= 0x0002 # bit 1 = u32 value offsets
header = serialize_header(
flags=flags,
entry_count=len(translations_ordered),
node_count=node_count,
edge_count=edge_count,
string_table_offset=string_table_offset,
string_table_size=len(string_table_bytes),
value_array_offset=value_array_offset,
)
# 11. Assemble
binary = header + edge_bytes + value_array_bytes + string_table_bytes
stats = {
'entry_count': len(translations_ordered),
'node_count': node_count,
'edge_count': edge_count,
'bits_per_edge': 50,
'edge_array_size': len(edge_bytes),
'value_array_size': len(value_array_bytes),
'string_table_size': len(string_table_bytes),
'raw_string_table_size': raw_table_size,
'total_size': len(binary),
'header_size': HEADER_SIZE,
'use_u32_offsets': use_u32,
'split_storage': split_storage,
}
return binary, stats, root, skip_cache, translations_ordered, parsed
def verify_compilation(binary_data, parsed_entries, root, skip_cache):
"""Verify compiled binary by deserializing and looking up every entry.
Returns (correct, wrong, missing).
"""
header = parse_header(binary_data)
entry_count = header['entry_count']
edge_count = header['edge_count']
use_u32 = bool(header['flags'] & 0x0002)
# Extract sections
edge_start = HEADER_SIZE
edge_end = header['value_array_offset']
edge_bytes = binary_data[edge_start:edge_end]
val_start = header['value_array_offset']
val_end = header['string_table_offset']
val_bytes = binary_data[val_start:val_end]
str_start = header['string_table_offset']
str_bytes = binary_data[str_start:]
# Deserialize
edges = deserialize_edges(edge_bytes, edge_count)
value_offsets = deserialize_value_array(val_bytes, entry_count, use_u32)
raw_table = decompress_string_table(str_bytes)
# Build adjacency from deserialized edges for lookup
# Reconstruct graph: node_id -> list of (stroke, target, skip)
adj = {}
node_finals = set()
edge_idx = 0
# We need to figure out which edges belong to which node.
# Edges are stored in DFS node order; is_last marks end of a node's edge list.
current_node = 0
node_edges = {}
i = 0
while i < len(edges):
stroke, target, skip, is_last = edges[i]
if current_node not in node_edges:
node_edges[current_node] = []
node_edges[current_node].append((stroke, target, skip))
if is_last:
current_node += 1
# Skip nodes that have no edges (they won't appear in edge list)
# We detect these by checking if next edge's parent should be higher
i += 1
# Now look up each entry via the reconstructed DAWG
correct = 0
wrong = 0
missing = 0
for strokes, expected in parsed_entries:
idx = dawg_lookup_index(root, strokes, skip_cache)
if idx < 0 or idx >= len(value_offsets):
missing += 1
continue
offset = value_offsets[idx]
# Find null terminator
try:
translation = lookup_string(raw_table, offset)
except (ValueError, IndexError):
missing += 1
continue
if translation == expected:
correct += 1
else:
wrong += 1
if wrong <= 5:
print(f" WRONG: strokes={strokes}, expected='{expected}', got='{translation}'")
return correct, wrong, missing
def print_stats(stats, target_kb=533):
"""Print compilation statistics."""
target_bytes = target_kb * 1024
total = stats['total_size']
pct = (total / target_bytes) * 100 if target_bytes else 0
print(f"Dictionary Compilation Stats:")
print(f" Entries: {stats['entry_count']:>10,}")
print(f" DAWG nodes: {stats['node_count']:>10,}")
print(f" DAWG edges: {stats['edge_count']:>10,}")
print(f" Bits/edge: {stats['bits_per_edge']:>10}")
print(f" ---")
print(f" Header: {stats['header_size']:>10,} bytes")
print(f" Edge array: {stats['edge_array_size']:>10,} bytes ({stats['edge_array_size']/1024:.1f} KB)")
print(f" Value array: {stats['value_array_size']:>10,} bytes ({stats['value_array_size']/1024:.1f} KB)")
print(f" String table: {stats['string_table_size']:>10,} bytes ({stats['string_table_size']/1024:.1f} KB)")
print(f" ---")
print(f" TOTAL: {total:>10,} bytes ({total/1024:.1f} KB)")
print(f" Budget: {target_bytes:>10,} bytes ({target_kb} KB)")
print(f" Usage: {pct:>9.1f}%")
if total <= target_bytes:
print(f" Status: FITS ({(target_bytes - total)/1024:.1f} KB remaining)")
else:
print(f" Status: OVER BUDGET by {(total - target_bytes)/1024:.1f} KB")
print(f" ---")
print(f" Split storage: {'yes' if stats['split_storage'] else 'no'}")
print(f" U32 offsets: {'yes' if stats['use_u32_offsets'] else 'no'}")
# ─── CLI ───
def main():
parser = argparse.ArgumentParser(
description='Compile steno dictionary to binary DAWG format')
parser.add_argument('input', nargs='?', default='/tmp/plover-main.json',
help='Path to JSON dictionary (default: /tmp/plover-main.json)')
parser.add_argument('--output', default='steno_dict.bin',
help='Output binary path (default: steno_dict.bin)')
parser.add_argument('--max-entries', type=int, default=120000,
help='Max entries to include (default: 120000)')
parser.add_argument('--target-size', type=int, default=462,
help='Target size in KB (default: 462)')
parser.add_argument('--split-storage', action='store_true',
help='Generate split-storage metadata header')
parser.add_argument('--stats', action='store_true',
help='Print detailed stats')
parser.add_argument('--verify', action='store_true',
help='Verify all entries after compilation')
args = parser.parse_args()
if not os.path.exists(args.input):
print(f"ERROR: Dictionary not found: {args.input}")
print("Download Plover dict:")
print(" curl -sL 'https://raw.githubusercontent.com/openstenoproject/plover/main/plover/assets/main.json' -o /tmp/plover-main.json")
sys.exit(1)
print(f"Compiling {args.input}...")
binary, stats, root, skip_cache, translations, parsed = \
compile_dictionary(args.input, args.max_entries, args.split_storage)
# Write output
with open(args.output, 'wb') as f:
f.write(binary)
print(f"Written {len(binary):,} bytes to {args.output}")
if args.stats:
print()
print_stats(stats, args.target_size)
if args.verify:
print()
print("Verifying...")
correct, wrong, missing = verify_compilation(binary, parsed, root, skip_cache)
total = correct + wrong + missing
print(f" Correct: {correct}/{total}")
print(f" Wrong: {wrong}/{total}")
print(f" Missing: {missing}/{total}")
if wrong > 0 or missing > 0:
print(" WARNING: Verification found errors!")
sys.exit(2)
else:
print(" All entries verified successfully.")
if __name__ == '__main__':
main()

106
tools/fetch_dict.py Normal file
View file

@ -0,0 +1,106 @@
#!/usr/bin/env python3
"""Download steno dictionary with caching.
Checks local file hash against known upstream hash.
If file exists and hash matches skip download.
If file missing or hash mismatch download fresh.
"""
import hashlib
import json
import os
import sys
import urllib.request
DICTS = {
"plover": {
"url": "https://raw.githubusercontent.com/openstenoproject/plover/main/plover/assets/main.json",
"filename": "plover-main.json",
},
"lapwing": {
"url": "https://raw.githubusercontent.com/aerickt/steno-dictionaries/main/lapwing-base.json",
"filename": "lapwing.json",
},
}
def sha256_file(path):
h = hashlib.sha256()
with open(path, "rb") as f:
for chunk in iter(lambda: f.read(65536), b''):
h.update(chunk)
return h.hexdigest()
def download(url, dest):
import subprocess
print(f"Downloading {url}...")
try:
urllib.request.urlretrieve(url, dest)
except urllib.error.URLError:
# Fallback to curl which uses system cert store
subprocess.check_call(["curl", "-sL", "-o", dest, url])
size = os.path.getsize(dest)
print(f"Downloaded {size:,} bytes to {dest}")
def fetch(dict_name, dest_dir):
if dict_name not in DICTS:
print(f"Unknown dict: {dict_name}. Available: {', '.join(DICTS.keys())}")
return 1
info = DICTS[dict_name]
dest = os.path.join(dest_dir, info["filename"])
hash_file = dest + ".sha256"
if os.path.exists(dest):
local_hash = sha256_file(dest)
if os.path.exists(hash_file):
with open(hash_file) as f:
cached_hash = f.read().strip()
if local_hash == cached_hash:
print(f"{dest} up to date (sha256={local_hash[:12]}...)")
return 0
try:
json.load(open(dest))
print(f"{dest} exists, valid JSON (sha256={local_hash[:12]}...)")
with open(hash_file, "w") as f:
f.write(local_hash)
return 0
except (json.JSONDecodeError, IOError):
print(f"{dest} corrupted, re-downloading")
os.makedirs(dest_dir, exist_ok=True)
download(info["url"], dest)
new_hash = sha256_file(dest)
with open(hash_file, "w") as f:
f.write(new_hash)
try:
with open(dest) as f:
d = json.load(f)
print(f"Verified: {len(d)} entries")
except (json.JSONDecodeError, IOError) as e:
print(f"WARNING: downloaded file invalid: {e}")
return 1
return 0
def main():
if len(sys.argv) < 2:
print(f"Usage: {sys.argv[0]} <plover|lapwing> [dest_dir]")
print(f" dest_dir defaults to ./dicts/")
sys.exit(1)
dict_name = sys.argv[1]
dest_dir = sys.argv[2] if len(sys.argv) > 2 else os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "dicts")
sys.exit(fetch(dict_name, dest_dir))
if __name__ == "__main__":
main()

5
tools/requirements.txt Normal file
View file

@ -0,0 +1,5 @@
# zmk-steno-engine dictionary compiler
# stdlib only — no external dependencies required
#
# Python >= 3.8
# Uses: json, struct, zlib, argparse, sys, os, collections, math

460
tools/test_compiler.py Normal file
View file

@ -0,0 +1,460 @@
#!/usr/bin/env python3
"""Tests for dict_compiler.py — DAWG dictionary compiler."""
import json
import os
import struct
import sys
import tempfile
import pytest
# Ensure tools/ is importable
sys.path.insert(0, os.path.dirname(__file__))
from dict_compiler import (
STENO_KEYS,
FLAG_SPLIT_STORAGE,
HEADER_SIZE,
MAGIC,
VERSION,
build_dawg,
build_string_table,
compile_dictionary,
compute_skip_counts,
dawg_lookup_index,
decompress_string_table,
deserialize_edges,
deserialize_value_array,
get_dawg_traversal_order,
lookup_string,
parse_header,
parse_stroke,
parse_stroke_string,
serialize_edges,
serialize_header,
serialize_value_array,
trim_entries,
verify_compilation,
)
# ─── Small test dictionaries ───
SMALL_DICT = {
"S": "is",
"T": "it",
"K": "can",
"W": "with",
"H": "had",
"R": "are",
"TPHO": "no",
"STPH": "then",
"KAT": "cat",
"TKOG": "dog",
}
MULTI_STROKE_DICT = {
"S": "is",
"T": "it",
"KPA/HROL": "{}{-|}",
"TPHO/WUPB": "no one",
"K": "can",
}
MEDIUM_DICT = {
"S": "is",
"T": "it",
"K": "can",
"W": "with",
"H": "had",
"R": "are",
"A": "a",
"O": "oh",
"E": "he",
"U": "you",
"TPHO": "no",
"STPH": "then",
"KAT": "cat",
"TKOG": "dog",
"HOUS": "house",
"TPHAEUPL": "name",
"HROS": "also",
"TKPWRAET": "great",
"SKEL": "school",
"PLAS": "place",
}
def _make_temp_dict(d):
"""Write dict to temp JSON file, return path."""
fd, path = tempfile.mkstemp(suffix='.json')
with os.fdopen(fd, 'w') as f:
json.dump(d, f)
return path
def _parse_and_sort(d):
"""Parse dict entries and sort by stroke tuple."""
entries = []
for stroke_str, translation in d.items():
strokes = parse_stroke_string(stroke_str)
entries.append((strokes, translation))
entries.sort(key=lambda x: x[0])
return entries
# ─── Tests ───
class TestParseStroke:
"""Test stroke parsing."""
def test_left_side_stph(self):
"""STPH → S + T + P + H left side bits."""
val = parse_stroke("STPH")
expected = STENO_KEYS['S-'] | STENO_KEYS['T-'] | STENO_KEYS['P-'] | STENO_KEYS['H-']
assert val == expected
def test_right_side_eurb(self):
"""EURB → E + U + R + B right side bits."""
val = parse_stroke("EURB")
expected = STENO_KEYS['-E'] | STENO_KEYS['-U'] | STENO_KEYS['-R'] | STENO_KEYS['-B']
assert val == expected
def test_single_s(self):
"""S → just S bit."""
val = parse_stroke("S")
assert val == STENO_KEYS['S-']
def test_number_bar(self):
"""#STPH → number + S + T + P + H."""
val = parse_stroke("#STPH")
expected = (STENO_KEYS['#'] | STENO_KEYS['S-'] | STENO_KEYS['T-'] |
STENO_KEYS['P-'] | STENO_KEYS['H-'])
assert val == expected
def test_vowels(self):
"""AO → A + O vowel bits."""
val = parse_stroke("AO")
expected = STENO_KEYS['A-'] | STENO_KEYS['O-']
assert val == expected
def test_full_stroke(self):
"""STKPWHR → all left consonants."""
val = parse_stroke("STKPWHR")
expected = (STENO_KEYS['S-'] | STENO_KEYS['T-'] | STENO_KEYS['K-'] |
STENO_KEYS['P-'] | STENO_KEYS['W-'] | STENO_KEYS['H-'] |
STENO_KEYS['R-'])
assert val == expected
def test_star(self):
"""*E → star + E."""
val = parse_stroke("*E")
expected = STENO_KEYS['*'] | STENO_KEYS['-E']
assert val == expected
def test_multi_stroke_parse(self):
"""KPA/HROL parses to two stroke bitmasks."""
strokes = parse_stroke_string("KPA/HROL")
assert len(strokes) == 2
# First stroke: K + P + A
assert strokes[0] == (STENO_KEYS['K-'] | STENO_KEYS['P-'] | STENO_KEYS['A-'])
class TestBuildDawgSmall:
"""Test DAWG construction with small dictionary."""
def test_node_compression(self):
"""DAWG should have fewer nodes than a plain trie (compression happening)."""
entries = _parse_and_sort(SMALL_DICT)
root, node_count, edge_count = build_dawg(entries)
# With 10 entries, DAWG should have fewer nodes than
# total path length (which would be ~15+ for a trie)
assert node_count < 15
assert node_count > 0
assert edge_count > 0
def test_all_entries_reachable(self):
"""All entries should be reachable via traversal."""
entries = _parse_and_sort(SMALL_DICT)
root, node_count, edge_count = build_dawg(entries)
# Check each entry can be traversed
for strokes, _trans in entries:
node = root
for stroke in strokes:
assert stroke in node.edges, f"Missing edge for stroke in {strokes}"
node = node.edges[stroke]
assert node.final, f"Node not final for {strokes}"
class TestDawgLookup:
"""Test DAWG lookup via skip-count traversal."""
def test_all_lookups_correct(self):
"""All entries should have unique sequential skip-count indices."""
entries = _parse_and_sort(SMALL_DICT)
root, _, _ = build_dawg(entries)
skip_cache = compute_skip_counts(root)
indices = []
for strokes, _ in entries:
idx = dawg_lookup_index(root, strokes, skip_cache)
assert idx >= 0, f"Lookup failed for {strokes}"
indices.append(idx)
# All indices should be unique
assert len(set(indices)) == len(indices), "Duplicate indices found"
# Indices should be 0..n-1
assert sorted(indices) == list(range(len(entries)))
def test_missing_entry_returns_neg(self):
"""Looking up a non-existent stroke should return -1."""
entries = _parse_and_sort(SMALL_DICT)
root, _, _ = build_dawg(entries)
skip_cache = compute_skip_counts(root)
# A stroke not in the dict
fake_strokes = (0xDEAD,)
idx = dawg_lookup_index(root, fake_strokes, skip_cache)
assert idx == -1
def test_traversal_order_matches(self):
"""Traversal order should match sorted entries."""
entries = _parse_and_sort(SMALL_DICT)
root, _, _ = build_dawg(entries)
traversal = get_dawg_traversal_order(root)
entry_paths = [strokes for strokes, _ in entries]
assert traversal == entry_paths
class TestBinaryRoundTrip:
"""Test compile → binary → deserialize → verify."""
def test_small_dict_round_trip(self):
"""Compile small dict, deserialize, verify all lookups."""
path = _make_temp_dict(SMALL_DICT)
try:
binary, stats, root, skip_cache, translations, parsed = \
compile_dictionary(path, max_entries=100)
assert stats['entry_count'] == len(SMALL_DICT)
assert stats['total_size'] == len(binary)
assert stats['total_size'] > HEADER_SIZE
# Verify header
header = parse_header(binary)
assert header['magic'] == MAGIC
assert header['version'] == VERSION
assert header['entry_count'] == len(SMALL_DICT)
# Verify round-trip
correct, wrong, missing = verify_compilation(binary, parsed, root, skip_cache)
assert correct == len(SMALL_DICT)
assert wrong == 0
assert missing == 0
finally:
os.unlink(path)
def test_medium_dict_round_trip(self):
"""Medium dict round-trip."""
path = _make_temp_dict(MEDIUM_DICT)
try:
binary, stats, root, skip_cache, translations, parsed = \
compile_dictionary(path, max_entries=100)
correct, wrong, missing = verify_compilation(binary, parsed, root, skip_cache)
assert correct == len(MEDIUM_DICT)
assert wrong == 0
assert missing == 0
finally:
os.unlink(path)
def test_edge_serialization_round_trip(self):
"""Edge bit-packing round-trip."""
entries = _parse_and_sort(SMALL_DICT)
root, node_count, edge_count = build_dawg(entries)
edge_bytes, node_id_map, skip_cache = serialize_edges(root, node_count, edge_count)
edges = deserialize_edges(edge_bytes, edge_count)
assert len(edges) == edge_count
# Each edge should have valid fields
for stroke, target, skip, is_last in edges:
assert 0 <= stroke <= 0xFFFF
assert 0 <= target <= 0xFFFF
assert 0 <= skip <= 0x1FFFF
def test_string_table_round_trip(self):
"""String table compress/decompress round-trip."""
translations = list(SMALL_DICT.values())
table_bytes, offsets, raw_size = build_string_table(translations)
raw = decompress_string_table(table_bytes)
for i, trans in enumerate(translations):
recovered = lookup_string(raw, offsets[i])
assert recovered == trans, f"Mismatch at {i}: '{trans}' vs '{recovered}'"
class TestEntryTrimming:
"""Test entry trimming logic."""
def test_trim_keeps_single_stroke(self):
"""With max_entries < total, single-stroke entries kept preferentially."""
# Build dict: 60 single-stroke, 60 multi-stroke
d = {}
# Single-stroke entries (using various left-hand keys)
single_keys = list("STKPWHRAO*EUFRPBLGTSDZ")
for i, k in enumerate(single_keys[:15]):
d[k] = f"word_{i}"
# Multi-stroke entries
for i in range(85):
d[f"KAT/TKOG/{i % 10}"] = f"long_translation_{i}"
# Total = 100 entries
entries = list(d.items())
assert len(entries) == 100
trimmed = trim_entries(entries, 50)
assert len(trimmed) == 50
# Count single vs multi in result
single_count = sum(1 for s, _ in trimmed if '/' not in s)
multi_count = sum(1 for s, _ in trimmed if '/' in s)
# All 15 single-stroke entries should be kept
assert single_count == 15
assert multi_count == 35
def test_no_trim_when_under_limit(self):
"""No trimming when entries < max_entries."""
entries = list(SMALL_DICT.items())
trimmed = trim_entries(entries, 1000)
assert len(trimmed) == len(entries)
def test_trim_multi_stroke_by_length(self):
"""Multi-stroke entries trimmed by translation length (shorter kept)."""
d = {"S": "is"} # 1 single-stroke
# Add multi-stroke with varying translation lengths
d["KAT/S"] = "ab" # short
d["KAT/T"] = "abcdefghij" # long
d["KAT/K"] = "abc" # medium
entries = list(d.items())
trimmed = trim_entries(entries, 3)
# Should keep: single("S"), then shortest multi-stroke
assert len(trimmed) == 3
trans = [t for _, t in trimmed]
assert "is" in trans # single stroke kept
assert "ab" in trans # shortest multi kept
assert "abc" in trans # medium kept
assert "abcdefghij" not in trans # longest dropped
class TestSplitStorageFlag:
"""Test --split-storage flag."""
def test_flag_set_in_header(self):
"""split_storage flag should set bit 0 in header flags."""
path = _make_temp_dict(SMALL_DICT)
try:
binary, stats, _, _, _, _ = compile_dictionary(
path, max_entries=100, split_storage=True)
header = parse_header(binary)
assert header['flags'] & FLAG_SPLIT_STORAGE != 0
assert stats['split_storage'] is True
finally:
os.unlink(path)
def test_flag_not_set_by_default(self):
"""split_storage flag should NOT be set by default."""
path = _make_temp_dict(SMALL_DICT)
try:
binary, stats, _, _, _, _ = compile_dictionary(
path, max_entries=100, split_storage=False)
header = parse_header(binary)
assert header['flags'] & FLAG_SPLIT_STORAGE == 0
assert stats['split_storage'] is False
finally:
os.unlink(path)
class TestMultiStroke:
"""Test multi-stroke entry handling."""
def test_multi_stroke_compile_and_lookup(self):
"""Multi-stroke entries (e.g. KPA/HROL) should compile and look up correctly."""
path = _make_temp_dict(MULTI_STROKE_DICT)
try:
binary, stats, root, skip_cache, translations, parsed = \
compile_dictionary(path, max_entries=100)
correct, wrong, missing = verify_compilation(binary, parsed, root, skip_cache)
assert correct == len(MULTI_STROKE_DICT)
assert wrong == 0
assert missing == 0
finally:
os.unlink(path)
def test_multi_stroke_traversal(self):
"""Multi-stroke entries should appear in correct traversal order."""
entries = _parse_and_sort(MULTI_STROKE_DICT)
root, _, _ = build_dawg(entries)
skip_cache = compute_skip_counts(root)
for strokes, _ in entries:
idx = dawg_lookup_index(root, strokes, skip_cache)
assert idx >= 0, f"Multi-stroke lookup failed: {strokes}"
def test_kpa_hrol_specific(self):
"""KPA/HROL → {}{-|} specifically."""
d = {"KPA/HROL": "{}{-|}"}
entries = _parse_and_sort(d)
root, _, _ = build_dawg(entries)
skip_cache = compute_skip_counts(root)
strokes = parse_stroke_string("KPA/HROL")
idx = dawg_lookup_index(root, strokes, skip_cache)
assert idx == 0 # only entry → index 0
class TestHeaderSerialization:
"""Test header pack/unpack."""
def test_header_size(self):
"""Header should be exactly 32 bytes."""
header = serialize_header(0, 100, 50, 200, 1000, 500, 800)
assert len(header) == HEADER_SIZE
def test_header_round_trip(self):
"""Header fields should survive pack/unpack."""
header = serialize_header(
flags=FLAG_SPLIT_STORAGE,
entry_count=12345,
node_count=6789,
edge_count=11111,
string_table_offset=99999,
string_table_size=55555,
value_array_offset=44444,
)
parsed = parse_header(header)
assert parsed['magic'] == MAGIC
assert parsed['version'] == VERSION
assert parsed['flags'] == FLAG_SPLIT_STORAGE
assert parsed['entry_count'] == 12345
assert parsed['node_count'] == 6789
assert parsed['edge_count'] == 11111
assert parsed['string_table_offset'] == 99999
assert parsed['string_table_size'] == 55555
assert parsed['value_array_offset'] == 44444
if __name__ == '__main__':
pytest.main([__file__, '-v'])

5
zephyr/module.yml Normal file
View file

@ -0,0 +1,5 @@
build:
cmake: .
kconfig: Kconfig
settings:
dts_root: .