Scaffolding: ZMK module structure, Kconfig, DTS, behavior driver
- zephyr/module.yml with dts_root for dt-bindings - Kconfig: STENO_ENGINE, dict selection (Plover/Lapwing/test), MPHF toggle, Unicode modes, history size, multi-stroke timeout - DTS behavior binding (one_param, steno key index) - behavior_steno.c: chord accumulation, all-up detection, multi-stroke buffering with timeout, star undo, formatter pipeline, 3-way dict dispatch (split/MPHF/simple trie) - dict_embed.S: .incbin from generated header path - steno_keys.h: 23-key layout, bit positions matching compiler
This commit is contained in:
parent
234bc8d731
commit
1ec204b845
33 changed files with 5443 additions and 0 deletions
6
.gitignore
vendored
6
.gitignore
vendored
|
|
@ -0,0 +1,6 @@
|
|||
__pycache__/
|
||||
*.pyc
|
||||
dicts/plover-main.json
|
||||
dicts/lapwing.json
|
||||
dicts/*.sha256
|
||||
tests/build/
|
||||
105
CMakeLists.txt
Normal file
105
CMakeLists.txt
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
# Copyright (c) 2024 Afiq Zudin Hadi
|
||||
# SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
|
||||
|
||||
if(CONFIG_STENO_ENGINE)
|
||||
if(NOT CONFIG_ZMK_SPLIT OR CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
|
||||
|
||||
target_sources(app PRIVATE
|
||||
src/behavior_steno.c
|
||||
src/output.c
|
||||
src/formatter.c
|
||||
src/undo.c
|
||||
src/dict_embed.S
|
||||
)
|
||||
|
||||
if(CONFIG_STENO_DICT_MPHF)
|
||||
target_sources(app PRIVATE src/dict_mphf.c)
|
||||
else()
|
||||
target_sources(app PRIVATE src/trie.c)
|
||||
endif()
|
||||
|
||||
target_include_directories(app PRIVATE
|
||||
include
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/src
|
||||
)
|
||||
|
||||
# ── Dictionary source resolution ──
|
||||
find_package(Python3 REQUIRED COMPONENTS Interpreter)
|
||||
set(STENO_DICT_BIN ${CMAKE_CURRENT_BINARY_DIR}/steno_dict.bin)
|
||||
set(STENO_DICTS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/dicts)
|
||||
set(STENO_FETCH ${CMAKE_CURRENT_SOURCE_DIR}/tools/fetch_dict.py)
|
||||
|
||||
if(CONFIG_STENO_DICT_PLOVER)
|
||||
set(STENO_DICT_NAME "plover")
|
||||
set(STENO_DICT_SRC ${STENO_DICTS_DIR}/plover-main.json)
|
||||
elseif(CONFIG_STENO_DICT_LAPWING)
|
||||
set(STENO_DICT_NAME "lapwing")
|
||||
set(STENO_DICT_SRC ${STENO_DICTS_DIR}/lapwing.json)
|
||||
else()
|
||||
set(STENO_DICT_SRC ${STENO_DICTS_DIR}/test.json)
|
||||
endif()
|
||||
|
||||
# Auto-download dict if needed (Plover/Lapwing only)
|
||||
if(DEFINED STENO_DICT_NAME AND NOT EXISTS ${STENO_DICT_SRC})
|
||||
message(STATUS "Steno: downloading ${STENO_DICT_NAME} dictionary...")
|
||||
execute_process(
|
||||
COMMAND ${Python3_EXECUTABLE} ${STENO_FETCH}
|
||||
${STENO_DICT_NAME} ${STENO_DICTS_DIR}
|
||||
RESULT_VARIABLE FETCH_RESULT
|
||||
)
|
||||
if(NOT FETCH_RESULT EQUAL 0)
|
||||
message(WARNING "Steno: dict download failed. Build may fail.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# ── Dictionary compilation ──
|
||||
if(EXISTS ${STENO_DICT_SRC})
|
||||
if(CONFIG_STENO_DICT_MPHF)
|
||||
# Fetch at build time if hash changed (re-run on rebuild)
|
||||
if(DEFINED STENO_DICT_NAME)
|
||||
add_custom_command(
|
||||
OUTPUT ${STENO_DICT_SRC}.stamp
|
||||
COMMAND ${Python3_EXECUTABLE} ${STENO_FETCH}
|
||||
${STENO_DICT_NAME} ${STENO_DICTS_DIR}
|
||||
COMMAND ${CMAKE_COMMAND} -E touch ${STENO_DICT_SRC}.stamp
|
||||
COMMENT "Checking ${STENO_DICT_NAME} dictionary for updates"
|
||||
)
|
||||
add_custom_target(steno_dict_fetch DEPENDS ${STENO_DICT_SRC}.stamp)
|
||||
endif()
|
||||
|
||||
add_custom_command(
|
||||
OUTPUT ${STENO_DICT_BIN}
|
||||
COMMAND ${Python3_EXECUTABLE}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_mphf.py
|
||||
${STENO_DICT_SRC} ${STENO_DICT_BIN}
|
||||
--max-size ${CONFIG_STENO_DICT_MAX_SIZE}
|
||||
DEPENDS ${STENO_DICT_SRC}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_mphf.py
|
||||
COMMENT "Compiling steno dictionary (MPHF)"
|
||||
)
|
||||
else()
|
||||
add_custom_command(
|
||||
OUTPUT ${STENO_DICT_BIN}
|
||||
COMMAND ${Python3_EXECUTABLE}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_simple.py
|
||||
${STENO_DICT_SRC} -o ${STENO_DICT_BIN}
|
||||
DEPENDS ${STENO_DICT_SRC}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_simple.py
|
||||
COMMENT "Compiling steno dictionary (simple)"
|
||||
)
|
||||
endif()
|
||||
|
||||
add_custom_target(steno_dict_gen DEPENDS ${STENO_DICT_BIN})
|
||||
if(TARGET steno_dict_fetch)
|
||||
add_dependencies(steno_dict_gen steno_dict_fetch)
|
||||
endif()
|
||||
add_dependencies(app steno_dict_gen)
|
||||
|
||||
# Generate header with dict path for .incbin
|
||||
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/steno_dict_path.h
|
||||
"#define STENO_DICT_BIN_PATH \"${STENO_DICT_BIN}\"\n")
|
||||
target_include_directories(app PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
|
||||
endif()
|
||||
|
||||
endif() # central role
|
||||
endif() # CONFIG_STENO_ENGINE
|
||||
85
Kconfig
Normal file
85
Kconfig
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
# Copyright (c) 2024 Afiq Zudin Hadi
|
||||
# SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
|
||||
|
||||
menuconfig STENO_ENGINE
|
||||
bool "Steno Engine"
|
||||
default n
|
||||
help
|
||||
Enable the stenography engine for ZMK.
|
||||
|
||||
if STENO_ENGINE
|
||||
|
||||
choice STENO_DICT
|
||||
prompt "Steno dictionary"
|
||||
default STENO_DICT_PLOVER
|
||||
|
||||
config STENO_DICT_PLOVER
|
||||
bool "Plover main dictionary (MPHF compressed)"
|
||||
help
|
||||
Use Plover main.json via MPHF compression (~44K entries in 453KB).
|
||||
|
||||
config STENO_DICT_LAPWING
|
||||
bool "Lapwing dictionary (MPHF compressed)"
|
||||
|
||||
config STENO_DICT_TEST
|
||||
bool "Test dictionary (46 entries, simple format)"
|
||||
help
|
||||
Small built-in test dictionary for development.
|
||||
|
||||
endchoice
|
||||
|
||||
config STENO_DICT_MPHF
|
||||
bool
|
||||
default y if STENO_DICT_PLOVER || STENO_DICT_LAPWING
|
||||
select ZLIB
|
||||
help
|
||||
Use MPHF (minimal perfect hash) dictionary format.
|
||||
Selects ZLIB for block-compressed string table decompression.
|
||||
Auto-selected for Plover/Lapwing dicts.
|
||||
|
||||
config STENO_CUSTOM_KEYMAP
|
||||
bool "Custom steno keymap"
|
||||
default n
|
||||
|
||||
choice STENO_UNICODE_MODE
|
||||
prompt "Unicode output mode"
|
||||
default STENO_UNICODE_MODE_NONE
|
||||
|
||||
config STENO_UNICODE_MODE_NONE
|
||||
bool "None (HID only)"
|
||||
|
||||
config STENO_UNICODE_MODE_LINUX
|
||||
bool "Linux (IBus)"
|
||||
|
||||
config STENO_UNICODE_MODE_MACOS
|
||||
bool "macOS"
|
||||
|
||||
config STENO_UNICODE_MODE_WINC
|
||||
bool "Windows (WinCompose)"
|
||||
|
||||
endchoice
|
||||
|
||||
config STENO_HISTORY_SIZE
|
||||
int "Stroke history size (undo depth)"
|
||||
default 100
|
||||
range 10 500
|
||||
|
||||
config STENO_KEY_DELAY_MS
|
||||
int "Key output delay (ms)"
|
||||
default 0
|
||||
range 0 50
|
||||
|
||||
config STENO_MULTI_STROKE_TIMEOUT_MS
|
||||
int "Multi-stroke timeout (ms)"
|
||||
default 500
|
||||
range 100 5000
|
||||
|
||||
config STENO_DICT_MAX_SIZE
|
||||
int "Max dictionary binary size (bytes)"
|
||||
default 430080
|
||||
help
|
||||
Max compiled dict size. 430080 = 420KB.
|
||||
Leaves room for zlib decompressor + USB logging overhead.
|
||||
The MPHF compiler auto-trims to fit.
|
||||
|
||||
endif # STENO_ENGINE
|
||||
132
LICENSE
Normal file
132
LICENSE
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
# PolyForm Noncommercial License 1.0.0
|
||||
|
||||
<https://polyformproject.org/licenses/noncommercial/1.0.0>
|
||||
|
||||
## Acceptance
|
||||
|
||||
In order to get any license under these terms, you must agree
|
||||
to them as both strict obligations and conditions to all
|
||||
your licenses.
|
||||
|
||||
## Copyright License
|
||||
|
||||
The licensor grants you a copyright license for the software
|
||||
to do everything you might do with the software that would
|
||||
otherwise infringe the licensor's copyright in it for any
|
||||
permitted purpose. However, you may only distribute the
|
||||
software according to [Distribution License](#distribution-license)
|
||||
and make changes or new works based on the software according
|
||||
to [Changes and New Works License](#changes-and-new-works-license).
|
||||
|
||||
## Distribution License
|
||||
|
||||
The licensor grants you an additional copyright license to
|
||||
distribute copies of the software. Your license to distribute
|
||||
covers distributing the software with changes and new works
|
||||
permitted by [Changes and New Works License](#changes-and-new-works-license).
|
||||
|
||||
## Notices
|
||||
|
||||
You must ensure that anyone who gets a copy of any part of
|
||||
the software from you also gets a copy of these terms or the
|
||||
URL for them above, as well as copies of any plain-text lines
|
||||
beginning with `Required Notice:` that the licensor provided
|
||||
with the software. For example:
|
||||
|
||||
> Required Notice: Copyright Afiq Zudin Hadi (https://github.com/afiqzudinhadi)
|
||||
|
||||
## Changes and New Works License
|
||||
|
||||
The licensor grants you an additional copyright license to
|
||||
make changes and new works based on the software for any
|
||||
permitted purpose.
|
||||
|
||||
## Patent License
|
||||
|
||||
The licensor grants you a patent license for the software that
|
||||
covers patent claims the licensor can license, or becomes able
|
||||
to license, that you would infringe by using the software.
|
||||
|
||||
## Noncommercial Purposes
|
||||
|
||||
Any noncommercial purpose is a permitted purpose.
|
||||
|
||||
## Personal Uses
|
||||
|
||||
Personal use for research, experiment, and testing for
|
||||
the benefit of public knowledge, personal study, private
|
||||
entertainment, hobby projects, amateur pursuits, or religious
|
||||
observance, without any anticipated commercial application,
|
||||
is use for a permitted purpose.
|
||||
|
||||
## Noncommercial Organizations
|
||||
|
||||
Use by any charitable organization, educational institution,
|
||||
public research organization, public safety or health
|
||||
organization, environmental protection organization, or
|
||||
government institution is use for a permitted purpose
|
||||
regardless of the source of funding or obligations resulting
|
||||
from the funding.
|
||||
|
||||
## Fair Use
|
||||
|
||||
You may have "fair use" rights for the software under the
|
||||
law. These terms do not limit them.
|
||||
|
||||
## No Other Rights
|
||||
|
||||
These terms do not allow you to sublicense or transfer any
|
||||
of your licenses to anyone else, or prevent the licensor
|
||||
from granting licenses to anyone else. These terms do not
|
||||
imply any other licenses.
|
||||
|
||||
## Patent Defense
|
||||
|
||||
If you make any written claim that the software infringes
|
||||
or contributes to infringement of any patent, your patent
|
||||
license for the software granted under these terms ends
|
||||
immediately. If your company makes such a claim, your
|
||||
patent license ends immediately for work on behalf of your
|
||||
company.
|
||||
|
||||
## Violations
|
||||
|
||||
The first time you are notified in writing that you have
|
||||
violated any of these terms, or done anything with the
|
||||
software not covered by your licenses, your licenses can
|
||||
nonetheless continue if you come into full compliance with
|
||||
these terms, and take practical steps to correct past
|
||||
violations, within 32 days of receiving notice. Otherwise,
|
||||
all your licenses end immediately.
|
||||
|
||||
## No Liability
|
||||
|
||||
As far as the law allows, the software comes as is, without
|
||||
any warranty or condition, and the licensor will not be
|
||||
liable to you for any damages arising out of these terms or
|
||||
the use or nature of the software, under any kind of legal
|
||||
claim.
|
||||
|
||||
## Definitions
|
||||
|
||||
The **licensor** is the individual or entity offering these
|
||||
terms, and the **software** is the software the licensor
|
||||
makes available under these terms.
|
||||
|
||||
**You** refers to the individual or entity agreeing to these
|
||||
terms.
|
||||
|
||||
**Your company** is any legal entity, sole proprietorship,
|
||||
or other kind of organization that you work for, plus all
|
||||
organizations that have control over, are under the control
|
||||
of, or are under common control with that organization.
|
||||
**Control** means ownership of substantially all the assets
|
||||
of an entity, or the power to direct its management and
|
||||
policies by vote, contract, or otherwise. Control can be
|
||||
direct or indirect.
|
||||
|
||||
**Your licenses** are all the licenses granted to you for
|
||||
the software under these terms.
|
||||
|
||||
**Use** means anything you do with the software requiring
|
||||
one of your licenses.
|
||||
66
README.md
Normal file
66
README.md
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
# zmk-steno-engine
|
||||
|
||||
Clean-room stenography engine for [ZMK Firmware](https://zmk.dev). Dictionary-based lookup with multi-stroke support, optimized for nRF52840 flash constraints.
|
||||
|
||||
**Status:** Early development — basic single/multi-stroke lookup works, formatter not yet implemented.
|
||||
|
||||
## Features
|
||||
|
||||
- Standard 23-key steno layout
|
||||
- Sorted-array dictionary with binary search lookup
|
||||
- Multi-stroke support with configurable timeout
|
||||
- All-up chord detection
|
||||
- HID keyboard output (ASCII)
|
||||
- Build-time dictionary compilation from Plover JSON
|
||||
- Test dictionary included for development
|
||||
|
||||
## Building
|
||||
|
||||
Add as a ZMK module in your `west.yml`:
|
||||
|
||||
```yaml
|
||||
manifest:
|
||||
remotes:
|
||||
- name: zmk-steno-engine
|
||||
url-base: https://github.com/afiqzudinhadi
|
||||
projects:
|
||||
- name: zmk-steno-engine
|
||||
remote: zmk-steno-engine
|
||||
revision: optimize-dict
|
||||
```
|
||||
|
||||
Enable in your `.conf`:
|
||||
|
||||
```
|
||||
CONFIG_STENO_ENGINE=y
|
||||
```
|
||||
|
||||
Use in your keymap:
|
||||
|
||||
```dts
|
||||
#include <dt-bindings/zmk/steno_keys.h>
|
||||
|
||||
/ {
|
||||
keymap {
|
||||
steno_layer {
|
||||
bindings = <
|
||||
&steno STENO_SL &steno STENO_TL &steno STENO_PL ...
|
||||
>;
|
||||
};
|
||||
};
|
||||
};
|
||||
```
|
||||
|
||||
## Dictionary Compiler
|
||||
|
||||
```bash
|
||||
# Compile test dictionary
|
||||
python3 tools/compile_simple.py dicts/test.json -o steno_dict.bin --stats
|
||||
|
||||
# Compile Plover dictionary (trimmed)
|
||||
python3 tools/compile_simple.py plover-main.json -o steno_dict.bin --max-entries 120000 --stats
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
[PolyForm Noncommercial 1.0.0](LICENSE)
|
||||
48
dicts/test.json
Normal file
48
dicts/test.json
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
{
|
||||
"S": "is",
|
||||
"T": "it",
|
||||
"-T": "the",
|
||||
"K": "can",
|
||||
"W": "with",
|
||||
"H": "had",
|
||||
"R": "are",
|
||||
"A": "a",
|
||||
"AOEU": "I",
|
||||
"TKPW": "go",
|
||||
"HRAO": "hello",
|
||||
"WORBGD": "world",
|
||||
"KAT": "cat",
|
||||
"TKOG": "dog",
|
||||
"HOUS": "house",
|
||||
"TPEUR": "fire",
|
||||
"WAURT": "water",
|
||||
"TPOOD": "food",
|
||||
"PWO*BG": "book",
|
||||
"KO": "co",
|
||||
"KO/PHAOURD": "computer",
|
||||
"KO/PHAOURDZ": "computers",
|
||||
"TEFT": "test",
|
||||
"TEFTD": "tested",
|
||||
"TEFTG": "testing",
|
||||
"STE/TPHO": "steno",
|
||||
"STE/TPHOG": "stenography",
|
||||
"KPA": "{}{-|}",
|
||||
"S-P": "{^ ^}",
|
||||
"TP-PL": "{.}",
|
||||
"KW-BG": "{,}",
|
||||
"H-F": "{?}",
|
||||
"SKHRAPL": "{!}",
|
||||
"*": "{*}",
|
||||
"TPHO": "no",
|
||||
"KWRA": "yes",
|
||||
"THA": "that",
|
||||
"PWUT": "but",
|
||||
"TPHOT": "not",
|
||||
"SREU": "have",
|
||||
"WHA": "what",
|
||||
"HEU": "he",
|
||||
"SHEU": "she",
|
||||
"THE": "they",
|
||||
"WE": "we",
|
||||
"KWREU": "I"
|
||||
}
|
||||
13
dts/behaviors/steno_engine.dtsi
Normal file
13
dts/behaviors/steno_engine.dtsi
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
/*
|
||||
* Copyright (c) 2024 Afiq Zudin Hadi
|
||||
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
|
||||
*/
|
||||
|
||||
/ {
|
||||
behaviors {
|
||||
/omit-if-no-ref/ steno: steno_engine {
|
||||
compatible = "zmk,behavior-steno-engine";
|
||||
#binding-cells = <1>;
|
||||
};
|
||||
};
|
||||
};
|
||||
14
dts/bindings/behaviors/zmk,behavior-steno-engine.yaml
Normal file
14
dts/bindings/behaviors/zmk,behavior-steno-engine.yaml
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
# Copyright (c) 2024 Afiq Zudin Hadi
|
||||
# SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
|
||||
|
||||
description: |
|
||||
Steno engine behavior. Maps physical keys to steno positions,
|
||||
detects chords on all-keys-up, and emits translated text.
|
||||
|
||||
Usage in keymap:
|
||||
&steno STENO_SL // left S key
|
||||
&steno STENO_TL // left T key
|
||||
|
||||
compatible: "zmk,behavior-steno-engine"
|
||||
|
||||
include: one_param.yaml
|
||||
53
include/dt-bindings/zmk/steno_keys.h
Normal file
53
include/dt-bindings/zmk/steno_keys.h
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
/*
|
||||
* Copyright (c) 2024 Afiq Zudin Hadi
|
||||
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
|
||||
*
|
||||
* Standard steno key layout. Each key = bit position in uint32_t chord.
|
||||
* Bit assignments match the compiler's STENO_KEYS bitmask exactly.
|
||||
*
|
||||
* Layout (standard steno order):
|
||||
* S T K P W H R A O * E U F R P B L G T S D Z #
|
||||
* 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22
|
||||
*/
|
||||
|
||||
#ifndef DT_BINDINGS_ZMK_STENO_KEYS_H_
|
||||
#define DT_BINDINGS_ZMK_STENO_KEYS_H_
|
||||
|
||||
/* Left hand consonants */
|
||||
#define STENO_SL 0 /* S- (left) */
|
||||
#define STENO_TL 1 /* T- */
|
||||
#define STENO_KL 2 /* K- */
|
||||
#define STENO_PL 3 /* P- */
|
||||
#define STENO_WL 4 /* W- */
|
||||
#define STENO_HL 5 /* H- */
|
||||
#define STENO_RL 6 /* R- (left) */
|
||||
|
||||
/* Vowels */
|
||||
#define STENO_A 7 /* A */
|
||||
#define STENO_O 8 /* O */
|
||||
|
||||
/* Center */
|
||||
#define STENO_STAR 9 /* * */
|
||||
|
||||
/* Vowels (right) */
|
||||
#define STENO_E 10 /* E */
|
||||
#define STENO_U 11 /* U */
|
||||
|
||||
/* Right hand consonants */
|
||||
#define STENO_FR 12 /* -F */
|
||||
#define STENO_RR 13 /* -R (right) */
|
||||
#define STENO_PR 14 /* -P (right) */
|
||||
#define STENO_BR 15 /* -B */
|
||||
#define STENO_LR 16 /* -L */
|
||||
#define STENO_GR 17 /* -G */
|
||||
#define STENO_TR 18 /* -T (right) */
|
||||
#define STENO_SR 19 /* -S (right) */
|
||||
#define STENO_DR 20 /* -D */
|
||||
#define STENO_ZR 21 /* -Z */
|
||||
|
||||
/* Number bar */
|
||||
#define STENO_NUM 22 /* # */
|
||||
|
||||
#define STENO_KEY_COUNT 23
|
||||
|
||||
#endif /* DT_BINDINGS_ZMK_STENO_KEYS_H_ */
|
||||
271
src/behavior_steno.c
Normal file
271
src/behavior_steno.c
Normal file
|
|
@ -0,0 +1,271 @@
|
|||
/*
|
||||
* Copyright (c) 2024 Afiq Zudin Hadi
|
||||
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
|
||||
*/
|
||||
|
||||
#define DT_DRV_COMPAT zmk_behavior_steno_engine
|
||||
|
||||
#include <zephyr/device.h>
|
||||
#include <zephyr/kernel.h>
|
||||
#include <zephyr/logging/log.h>
|
||||
|
||||
#include <drivers/behavior.h>
|
||||
#include <zmk/behavior.h>
|
||||
|
||||
#include "output.h"
|
||||
#include "undo.h"
|
||||
#include "formatter.h"
|
||||
|
||||
#if IS_ENABLED(CONFIG_STENO_DICT_MPHF)
|
||||
#include "dict_mphf.h"
|
||||
#else
|
||||
#include "trie.h"
|
||||
#endif
|
||||
|
||||
LOG_MODULE_DECLARE(zmk, CONFIG_ZMK_LOG_LEVEL);
|
||||
|
||||
extern const uint8_t _steno_dict_start[];
|
||||
extern const uint8_t _steno_dict_end[];
|
||||
|
||||
#if IS_ENABLED(CONFIG_STENO_DICT_MPHF)
|
||||
static struct dict_mphf mphf_dict;
|
||||
#endif
|
||||
|
||||
#define STENO_MAX_MULTI 8
|
||||
#define STENO_MULTI_TIMEOUT_MS CONFIG_STENO_MULTI_STROKE_TIMEOUT_MS
|
||||
|
||||
static inline const char *dict_lookup(const uint32_t *strokes, uint8_t count)
|
||||
{
|
||||
#if IS_ENABLED(CONFIG_STENO_DICT_MPHF)
|
||||
return dict_mphf_lookup(&mphf_dict, strokes, count);
|
||||
#else
|
||||
return steno_trie_lookup(strokes, count);
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline bool dict_has_prefix(const uint32_t *strokes, uint8_t count)
|
||||
{
|
||||
#if IS_ENABLED(CONFIG_STENO_DICT_MPHF)
|
||||
return (count == 1) ? dict_mphf_has_prefix(&mphf_dict, strokes[0]) : false;
|
||||
#else
|
||||
return steno_trie_has_prefix(strokes, count);
|
||||
#endif
|
||||
}
|
||||
|
||||
struct steno_state {
|
||||
uint32_t current_chord;
|
||||
uint8_t keys_held;
|
||||
uint32_t pending_strokes[STENO_MAX_MULTI];
|
||||
uint8_t stroke_count;
|
||||
struct k_work_delayable multi_timeout;
|
||||
};
|
||||
|
||||
static struct steno_state state;
|
||||
static struct steno_fmt_state fmt_state;
|
||||
static struct stroke_history undo_history;
|
||||
static bool dict_ready;
|
||||
|
||||
static void flush_strokes(void);
|
||||
static void multi_timeout_handler(struct k_work *work);
|
||||
|
||||
static void emit_formatted(const char *translation,
|
||||
const uint32_t *strokes, uint8_t stroke_count)
|
||||
{
|
||||
struct steno_fmt_result result;
|
||||
steno_fmt_process(&fmt_state, translation, &result);
|
||||
if (result.backspaces > 0) {
|
||||
steno_output_backspace(result.backspaces);
|
||||
}
|
||||
if (result.len > 0) {
|
||||
steno_output_send(result.text, result.len);
|
||||
}
|
||||
if (!result.is_command_only) {
|
||||
steno_undo_push(&undo_history, strokes, stroke_count,
|
||||
result.len + result.backspaces, 0, 0);
|
||||
}
|
||||
}
|
||||
|
||||
static void process_chord(void)
|
||||
{
|
||||
if (state.current_chord == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
/* Star-only stroke (bit 9) → undo */
|
||||
if (state.current_chord == (1U << 9)) {
|
||||
state.current_chord = 0;
|
||||
struct stroke_history_entry *ue = steno_undo_pop(&undo_history);
|
||||
if (ue) {
|
||||
steno_output_backspace(ue->output_len);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
k_work_cancel_delayable(&state.multi_timeout);
|
||||
|
||||
if (state.stroke_count < STENO_MAX_MULTI) {
|
||||
state.pending_strokes[state.stroke_count] = state.current_chord;
|
||||
state.stroke_count++;
|
||||
} else {
|
||||
flush_strokes();
|
||||
state.pending_strokes[0] = state.current_chord;
|
||||
state.stroke_count = 1;
|
||||
}
|
||||
state.current_chord = 0;
|
||||
|
||||
if (!dict_ready) {
|
||||
LOG_WRN("steno dict not ready, flushing");
|
||||
flush_strokes();
|
||||
return;
|
||||
}
|
||||
|
||||
const char *translation = dict_lookup(
|
||||
state.pending_strokes, state.stroke_count);
|
||||
LOG_INF("steno lookup %u strokes → %s", state.stroke_count,
|
||||
translation ? translation : "(null)");
|
||||
|
||||
if (translation) {
|
||||
emit_formatted(translation, state.pending_strokes, state.stroke_count);
|
||||
state.stroke_count = 0;
|
||||
return;
|
||||
}
|
||||
|
||||
if (dict_has_prefix(state.pending_strokes, state.stroke_count)) {
|
||||
k_work_schedule(&state.multi_timeout,
|
||||
K_MSEC(STENO_MULTI_TIMEOUT_MS));
|
||||
return;
|
||||
}
|
||||
|
||||
if (state.stroke_count > 1) {
|
||||
uint32_t last = state.pending_strokes[state.stroke_count - 1];
|
||||
state.stroke_count--;
|
||||
|
||||
const char *partial = dict_lookup(
|
||||
state.pending_strokes, state.stroke_count);
|
||||
if (partial) {
|
||||
emit_formatted(partial, state.pending_strokes, state.stroke_count);
|
||||
}
|
||||
|
||||
state.pending_strokes[0] = last;
|
||||
state.stroke_count = 1;
|
||||
|
||||
const char *rest = dict_lookup(&last, 1);
|
||||
if (rest) {
|
||||
emit_formatted(rest, &last, 1);
|
||||
state.stroke_count = 0;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
flush_strokes();
|
||||
}
|
||||
|
||||
static void flush_strokes(void)
|
||||
{
|
||||
LOG_DBG("Flushing %u untranslated strokes", state.stroke_count);
|
||||
state.stroke_count = 0;
|
||||
}
|
||||
|
||||
static void multi_timeout_handler(struct k_work *work)
|
||||
{
|
||||
ARG_UNUSED(work);
|
||||
|
||||
if (state.stroke_count == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
const char *translation = dict_lookup(
|
||||
state.pending_strokes, state.stroke_count);
|
||||
|
||||
if (translation) {
|
||||
emit_formatted(translation, state.pending_strokes, state.stroke_count);
|
||||
}
|
||||
state.stroke_count = 0;
|
||||
}
|
||||
|
||||
static int on_steno_binding_pressed(struct zmk_behavior_binding *binding,
|
||||
struct zmk_behavior_binding_event event)
|
||||
{
|
||||
uint32_t key_index = binding->param1;
|
||||
|
||||
if (key_index > 35) {
|
||||
return -EINVAL;
|
||||
}
|
||||
|
||||
state.current_chord |= (1U << key_index);
|
||||
state.keys_held++;
|
||||
|
||||
LOG_INF("steno press key=%u chord=0x%06X held=%u",
|
||||
key_index, state.current_chord, state.keys_held);
|
||||
|
||||
return ZMK_BEHAVIOR_OPAQUE;
|
||||
}
|
||||
|
||||
static int on_steno_binding_released(struct zmk_behavior_binding *binding,
|
||||
struct zmk_behavior_binding_event event)
|
||||
{
|
||||
if (state.keys_held > 0) {
|
||||
state.keys_held--;
|
||||
}
|
||||
|
||||
LOG_INF("steno release held=%u chord=0x%06X", state.keys_held, state.current_chord);
|
||||
|
||||
if (state.keys_held == 0 && state.current_chord != 0) {
|
||||
LOG_INF("steno all-up → process chord 0x%06X", state.current_chord);
|
||||
process_chord();
|
||||
}
|
||||
|
||||
return ZMK_BEHAVIOR_OPAQUE;
|
||||
}
|
||||
|
||||
static int behavior_steno_init(const struct device *dev)
|
||||
{
|
||||
ARG_UNUSED(dev);
|
||||
|
||||
state.current_chord = 0;
|
||||
state.keys_held = 0;
|
||||
state.stroke_count = 0;
|
||||
|
||||
steno_fmt_init(&fmt_state);
|
||||
steno_undo_init(&undo_history);
|
||||
k_work_init_delayable(&state.multi_timeout, multi_timeout_handler);
|
||||
|
||||
size_t dict_size = _steno_dict_end - _steno_dict_start;
|
||||
if (dict_size > 4) {
|
||||
int ret;
|
||||
#if IS_ENABLED(CONFIG_STENO_DICT_MPHF)
|
||||
ret = dict_mphf_init(&mphf_dict, _steno_dict_start, dict_size);
|
||||
#else
|
||||
ret = steno_trie_init(_steno_dict_start, dict_size);
|
||||
#endif
|
||||
if (ret == 0) {
|
||||
dict_ready = true;
|
||||
LOG_INF("Steno dict loaded (%u bytes)", (unsigned)dict_size);
|
||||
} else {
|
||||
LOG_ERR("Steno dict init failed: %d", ret);
|
||||
}
|
||||
} else {
|
||||
LOG_WRN("No steno dict embedded");
|
||||
}
|
||||
|
||||
LOG_INF("Steno engine initialized");
|
||||
return 0;
|
||||
}
|
||||
|
||||
static const struct behavior_driver_api steno_driver_api = {
|
||||
.locality = BEHAVIOR_LOCALITY_CENTRAL,
|
||||
.binding_pressed = on_steno_binding_pressed,
|
||||
.binding_released = on_steno_binding_released,
|
||||
};
|
||||
|
||||
#define STENO_INST(n) \
|
||||
BEHAVIOR_DT_INST_DEFINE(n, \
|
||||
behavior_steno_init, \
|
||||
NULL, \
|
||||
NULL, \
|
||||
NULL, \
|
||||
POST_KERNEL, \
|
||||
CONFIG_KERNEL_INIT_PRIORITY_DEFAULT, \
|
||||
&steno_driver_api);
|
||||
|
||||
DT_INST_FOREACH_STATUS_OKAY(STENO_INST)
|
||||
14
src/dict_embed.S
Normal file
14
src/dict_embed.S
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
#include "steno_dict_path.h"
|
||||
|
||||
.section .rodata.steno_dict, "a", %progbits
|
||||
.global _steno_dict_start
|
||||
.global _steno_dict_end
|
||||
|
||||
.balign 4
|
||||
_steno_dict_start:
|
||||
#ifdef STENO_DICT_BIN_PATH
|
||||
.incbin STENO_DICT_BIN_PATH
|
||||
#else
|
||||
.byte 0x00, 0x00, 0x00, 0x00
|
||||
#endif
|
||||
_steno_dict_end:
|
||||
323
src/dict_mphf.c
Normal file
323
src/dict_mphf.c
Normal file
|
|
@ -0,0 +1,323 @@
|
|||
/**
|
||||
* MPHF dictionary lookup engine — implementation.
|
||||
*
|
||||
* All data is read directly from flash. No heap allocation.
|
||||
* String table is block-compressed with zlib; decompressed on-demand
|
||||
* into a static 4KB buffer.
|
||||
*
|
||||
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
|
||||
*/
|
||||
|
||||
#include "dict_mphf.h"
|
||||
#include <string.h>
|
||||
|
||||
#ifdef __ZEPHYR__
|
||||
#include <zephyr/sys/crc.h>
|
||||
#endif
|
||||
|
||||
/* ─── Minimal inflate for non-Zephyr (native tests) ─── */
|
||||
|
||||
#ifndef __ZEPHYR__
|
||||
#include <stdlib.h>
|
||||
|
||||
/* Use zlib on host for native tests */
|
||||
#ifdef HAS_ZLIB
|
||||
#include <zlib.h>
|
||||
static int block_inflate(const uint8_t *src, size_t src_len,
|
||||
uint8_t *dst, size_t dst_cap, size_t *dst_len)
|
||||
{
|
||||
uLongf out_len = dst_cap;
|
||||
int ret = uncompress(dst, &out_len, src, src_len);
|
||||
if (ret == Z_OK) {
|
||||
*dst_len = out_len;
|
||||
return 0;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
#else
|
||||
/* Minimal tinf inflate — bundled for host-only testing.
|
||||
* On Zephyr, we use the kernel's built-in zlib. */
|
||||
#include "tinf/tinf.h"
|
||||
static int block_inflate(const uint8_t *src, size_t src_len,
|
||||
uint8_t *dst, size_t dst_cap, size_t *dst_len)
|
||||
{
|
||||
unsigned int out_len = dst_cap;
|
||||
/* Skip 2-byte zlib header, strip 4-byte adler32 checksum */
|
||||
if (src_len < 6) return -1;
|
||||
int ret = tinf_uncompress(dst, &out_len, src + 2, src_len - 6);
|
||||
if (ret == 0) {
|
||||
*dst_len = out_len;
|
||||
return 0;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
#endif /* HAS_ZLIB */
|
||||
|
||||
#else /* __ZEPHYR__ */
|
||||
|
||||
#include <zephyr/sys/util.h>
|
||||
|
||||
/* Zephyr built-in zlib decompression */
|
||||
#if __has_include(<zephyr/lib/zlib/zlib.h>)
|
||||
#include <zephyr/lib/zlib/zlib.h>
|
||||
#elif __has_include(<zlib.h>)
|
||||
#include <zlib.h>
|
||||
#endif
|
||||
|
||||
static int block_inflate(const uint8_t *src, size_t src_len,
|
||||
uint8_t *dst, size_t dst_cap, size_t *dst_len)
|
||||
{
|
||||
/* Try Zephyr's tinycrypt/miniz or fall back to raw copy */
|
||||
#if defined(CONFIG_ZLIB)
|
||||
uLongf out_len = dst_cap;
|
||||
int ret = uncompress(dst, &out_len, src, src_len);
|
||||
if (ret == Z_OK) {
|
||||
*dst_len = out_len;
|
||||
return 0;
|
||||
}
|
||||
return -1;
|
||||
#else
|
||||
/* No zlib available — strings must be uncompressed (flags bit 0 clear) */
|
||||
(void)src; (void)src_len; (void)dst; (void)dst_cap; (void)dst_len;
|
||||
return -1;
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif /* __ZEPHYR__ */
|
||||
|
||||
|
||||
/* ─── FNV-1a 32-bit hash ─── */
|
||||
|
||||
static uint32_t fnv1a_32(const uint8_t *data, size_t len)
|
||||
{
|
||||
uint32_t h = 0x811c9dc5u;
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
h ^= data[i];
|
||||
h *= 0x01000193u;
|
||||
}
|
||||
return h;
|
||||
}
|
||||
|
||||
static uint32_t hash_key(const uint8_t *key, size_t key_len, uint32_t seed)
|
||||
{
|
||||
uint32_t h = 0x811c9dc5u;
|
||||
uint8_t seed_bytes[4];
|
||||
seed_bytes[0] = (uint8_t)(seed);
|
||||
seed_bytes[1] = (uint8_t)(seed >> 8);
|
||||
seed_bytes[2] = (uint8_t)(seed >> 16);
|
||||
seed_bytes[3] = (uint8_t)(seed >> 24);
|
||||
for (int i = 0; i < 4; i++) {
|
||||
h ^= seed_bytes[i];
|
||||
h *= 0x01000193u;
|
||||
}
|
||||
for (size_t i = 0; i < key_len; i++) {
|
||||
h ^= key[i];
|
||||
h *= 0x01000193u;
|
||||
}
|
||||
return h;
|
||||
}
|
||||
|
||||
/* ─── Bit-packed field reading ─── */
|
||||
|
||||
static uint32_t read_bits(const uint8_t *data, uint32_t bit_pos, uint8_t n_bits)
|
||||
{
|
||||
uint32_t value = 0;
|
||||
for (uint8_t i = 0; i < n_bits; i++) {
|
||||
uint32_t byte_idx = (bit_pos + i) / 8;
|
||||
uint8_t bit_idx = (bit_pos + i) % 8;
|
||||
if (data[byte_idx] & (1u << bit_idx)) {
|
||||
value |= (1u << i);
|
||||
}
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
static inline uint32_t align4(uint32_t n)
|
||||
{
|
||||
return (n + 3u) & ~3u;
|
||||
}
|
||||
|
||||
/* ─── Init ─── */
|
||||
|
||||
int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len)
|
||||
{
|
||||
if (!dict || !data) return -1;
|
||||
if (len < sizeof(struct dict_mphf_header)) return -2;
|
||||
|
||||
const struct dict_mphf_header *hdr = (const struct dict_mphf_header *)data;
|
||||
if (hdr->magic != DICT_MPHF_MAGIC) return -3;
|
||||
if (hdr->version != DICT_MPHF_VERSION) return -4;
|
||||
|
||||
dict->header = hdr;
|
||||
const uint8_t *base = (const uint8_t *)data;
|
||||
uint32_t offset = sizeof(struct dict_mphf_header);
|
||||
|
||||
/* Displacements */
|
||||
dict->displacements = base + offset;
|
||||
uint32_t disp_bits_total = (uint32_t)hdr->bucket_count * hdr->disp_bits;
|
||||
dict->disp_section_len = align4((disp_bits_total + 7) / 8);
|
||||
offset += dict->disp_section_len;
|
||||
|
||||
/* Values */
|
||||
dict->values = base + offset;
|
||||
uint32_t val_bits_total = (uint32_t)hdr->entry_count * hdr->value_bits;
|
||||
dict->val_section_len = align4((val_bits_total + 7) / 8);
|
||||
offset += dict->val_section_len;
|
||||
|
||||
/* Fingerprints */
|
||||
dict->fingerprints = base + offset;
|
||||
dict->fp_section_len = align4(hdr->entry_count);
|
||||
offset += dict->fp_section_len;
|
||||
|
||||
/* String offsets (u24 LE, 3 bytes each) */
|
||||
dict->string_offsets = base + offset;
|
||||
offset += hdr->unique_count * 3;
|
||||
|
||||
/* String data section */
|
||||
dict->str_data_start = base + offset;
|
||||
|
||||
if (hdr->flags & DICT_MPHF_FLAG_COMPRESSED) {
|
||||
/* Block directory: u16 block_count + u32[] offsets */
|
||||
dict->block_count = dict->str_data_start[0] |
|
||||
((uint16_t)dict->str_data_start[1] << 8);
|
||||
dict->block_dir = (const uint32_t *)(dict->str_data_start + 2);
|
||||
dict->blocks_start = dict->str_data_start + 2 + dict->block_count * 4;
|
||||
} else {
|
||||
dict->block_count = 0;
|
||||
dict->block_dir = NULL;
|
||||
dict->blocks_start = dict->str_data_start;
|
||||
}
|
||||
|
||||
/* Prefix table at end */
|
||||
uint32_t prefix_bytes = (uint32_t)hdr->prefix_count * 4;
|
||||
if (len >= prefix_bytes) {
|
||||
dict->prefix_table = (const uint32_t *)(base + len - prefix_bytes);
|
||||
} else {
|
||||
dict->prefix_table = NULL;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ─── String decompression ─── */
|
||||
|
||||
static const char *resolve_string(const struct dict_mphf *dict, uint32_t val_id)
|
||||
{
|
||||
const uint8_t *off_ptr = dict->string_offsets + val_id * 3;
|
||||
uint32_t str_offset = (uint32_t)off_ptr[0]
|
||||
| ((uint32_t)off_ptr[1] << 8)
|
||||
| ((uint32_t)off_ptr[2] << 16);
|
||||
|
||||
if (!(dict->header->flags & DICT_MPHF_FLAG_COMPRESSED)) {
|
||||
return (const char *)(dict->str_data_start + str_offset);
|
||||
}
|
||||
|
||||
/* Block-compressed: decompress the right block */
|
||||
uint32_t block_idx = str_offset / DICT_MPHF_BLOCK_SIZE;
|
||||
uint32_t in_block_off = str_offset % DICT_MPHF_BLOCK_SIZE;
|
||||
|
||||
if (block_idx >= dict->block_count) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* Get compressed block bounds */
|
||||
uint32_t blk_start = dict->block_dir[block_idx];
|
||||
uint32_t blk_end;
|
||||
if (block_idx + 1 < dict->block_count) {
|
||||
blk_end = dict->block_dir[block_idx + 1];
|
||||
} else {
|
||||
/* Last block: extends to prefix_table or end of file */
|
||||
blk_end = (const uint8_t *)dict->prefix_table - dict->blocks_start;
|
||||
}
|
||||
|
||||
const uint8_t *compressed = dict->blocks_start + blk_start;
|
||||
uint32_t compressed_len = blk_end - blk_start;
|
||||
|
||||
static uint8_t decomp_buf[DICT_MPHF_BLOCK_SIZE];
|
||||
static uint32_t cached_block = UINT32_MAX;
|
||||
static size_t cached_len;
|
||||
|
||||
if (cached_block != block_idx) {
|
||||
size_t out_len;
|
||||
if (block_inflate(compressed, compressed_len,
|
||||
decomp_buf, sizeof(decomp_buf), &out_len) != 0) {
|
||||
return NULL;
|
||||
}
|
||||
cached_block = block_idx;
|
||||
cached_len = out_len;
|
||||
}
|
||||
|
||||
if (in_block_off >= cached_len) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return (const char *)(decomp_buf + in_block_off);
|
||||
}
|
||||
|
||||
/* ─── Lookup ─── */
|
||||
|
||||
const char *dict_mphf_lookup(const struct dict_mphf *dict,
|
||||
const uint32_t *strokes, uint8_t count)
|
||||
{
|
||||
if (!dict || !dict->header || !strokes || count == 0) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
const struct dict_mphf_header *hdr = dict->header;
|
||||
|
||||
uint8_t key_buf[32];
|
||||
size_t key_len = (size_t)count * 4;
|
||||
if (key_len > sizeof(key_buf)) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
for (uint8_t i = 0; i < count; i++) {
|
||||
key_buf[i * 4 + 0] = (uint8_t)(strokes[i]);
|
||||
key_buf[i * 4 + 1] = (uint8_t)(strokes[i] >> 8);
|
||||
key_buf[i * 4 + 2] = (uint8_t)(strokes[i] >> 16);
|
||||
key_buf[i * 4 + 3] = (uint8_t)(strokes[i] >> 24);
|
||||
}
|
||||
|
||||
uint32_t bucket = hash_key(key_buf, key_len, 0) % hdr->bucket_count;
|
||||
uint32_t d = read_bits(dict->displacements,
|
||||
bucket * (uint32_t)hdr->disp_bits,
|
||||
hdr->disp_bits);
|
||||
uint32_t slot = hash_key(key_buf, key_len, d + 1) % hdr->entry_count;
|
||||
|
||||
uint8_t expected_fp = (uint8_t)(fnv1a_32(key_buf, key_len) & 0xFF);
|
||||
if (dict->fingerprints[slot] != expected_fp) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
uint32_t val_id = read_bits(dict->values,
|
||||
slot * (uint32_t)hdr->value_bits,
|
||||
hdr->value_bits);
|
||||
if (val_id >= hdr->unique_count) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return resolve_string(dict, val_id);
|
||||
}
|
||||
|
||||
/* ─── has_prefix ─── */
|
||||
|
||||
bool dict_mphf_has_prefix(const struct dict_mphf *dict, uint32_t stroke)
|
||||
{
|
||||
if (!dict || !dict->header || !dict->prefix_table ||
|
||||
dict->header->prefix_count == 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
uint32_t lo = 0;
|
||||
uint32_t hi = dict->header->prefix_count;
|
||||
|
||||
while (lo < hi) {
|
||||
uint32_t mid = lo + (hi - lo) / 2;
|
||||
uint32_t val = dict->prefix_table[mid];
|
||||
if (val == stroke) return true;
|
||||
if (val < stroke) lo = mid + 1;
|
||||
else hi = mid;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
69
src/dict_mphf.h
Normal file
69
src/dict_mphf.h
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
/**
|
||||
* MPHF (Minimal Perfect Hash Function) dictionary lookup engine.
|
||||
*
|
||||
* Binary format v2: CHD MPHF + bit-packed displacements/values +
|
||||
* fingerprinted verification + block-compressed string table.
|
||||
*
|
||||
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
|
||||
*/
|
||||
|
||||
#ifndef DICT_MPHF_H
|
||||
#define DICT_MPHF_H
|
||||
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
|
||||
#define DICT_MPHF_MAGIC 0x4F4E5453 /* "STNO" */
|
||||
#define DICT_MPHF_VERSION 2
|
||||
|
||||
#define DICT_MPHF_FLAG_COMPRESSED 0x0001
|
||||
|
||||
#define DICT_MPHF_BLOCK_SIZE 4096
|
||||
|
||||
struct dict_mphf_header {
|
||||
uint32_t magic;
|
||||
uint16_t version;
|
||||
uint16_t flags;
|
||||
uint32_t entry_count;
|
||||
uint32_t bucket_count;
|
||||
uint32_t unique_count;
|
||||
uint8_t value_bits;
|
||||
uint8_t disp_bits;
|
||||
uint16_t prefix_count;
|
||||
uint32_t reserved0;
|
||||
uint32_t reserved1;
|
||||
} __attribute__((packed));
|
||||
|
||||
_Static_assert(sizeof(struct dict_mphf_header) == 32, "header must be 32 bytes");
|
||||
|
||||
struct dict_mphf {
|
||||
const struct dict_mphf_header *header;
|
||||
const uint8_t *displacements;
|
||||
const uint8_t *values;
|
||||
const uint8_t *fingerprints;
|
||||
const uint8_t *string_offsets;
|
||||
const uint8_t *str_data_start; /* start of string data section */
|
||||
const uint32_t *prefix_table;
|
||||
uint32_t disp_section_len;
|
||||
uint32_t val_section_len;
|
||||
uint32_t fp_section_len;
|
||||
uint16_t block_count;
|
||||
const uint32_t *block_dir; /* block offset directory */
|
||||
const uint8_t *blocks_start; /* start of compressed blocks */
|
||||
uint32_t str_data_len; /* total string data section length */
|
||||
};
|
||||
|
||||
int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len);
|
||||
|
||||
const char *dict_mphf_lookup(const struct dict_mphf *dict,
|
||||
const uint32_t *strokes, uint8_t count);
|
||||
|
||||
bool dict_mphf_has_prefix(const struct dict_mphf *dict, uint32_t stroke);
|
||||
|
||||
static inline uint32_t dict_mphf_count(const struct dict_mphf *dict)
|
||||
{
|
||||
return dict->header->entry_count;
|
||||
}
|
||||
|
||||
#endif /* DICT_MPHF_H */
|
||||
335
src/formatter.c
Normal file
335
src/formatter.c
Normal file
|
|
@ -0,0 +1,335 @@
|
|||
/*
|
||||
* Copyright (c) 2024 Afiq Zudin Hadi
|
||||
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
|
||||
*
|
||||
* Plover formatting engine — parses translation strings containing
|
||||
* {commands} and produces clean output with spacing/capitalization.
|
||||
* No malloc — all stack/static.
|
||||
*/
|
||||
|
||||
#include "formatter.h"
|
||||
#include <string.h>
|
||||
|
||||
/* ── helpers ────────────────────────────────────────────────────── */
|
||||
|
||||
static inline bool is_upper(char c) { return c >= 'A' && c <= 'Z'; }
|
||||
static inline bool is_lower(char c) { return c >= 'a' && c <= 'z'; }
|
||||
static inline char to_upper(char c) { return is_lower(c) ? (char)(c - 32) : c; }
|
||||
static inline char to_lower(char c) { return is_upper(c) ? (char)(c + 32) : c; }
|
||||
|
||||
static bool str_eq(const char *a, const char *b, size_t n)
|
||||
{
|
||||
for (size_t i = 0; i < n; i++) {
|
||||
if (a[i] != b[i]) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
static bool starts_with(const char *s, size_t slen, const char *prefix)
|
||||
{
|
||||
size_t plen = strlen(prefix);
|
||||
if (slen < plen) return false;
|
||||
return str_eq(s, prefix, plen);
|
||||
}
|
||||
|
||||
/* ── output buffer append ───────────────────────────────────────── */
|
||||
|
||||
static void emit_char(struct steno_fmt_result *r, char c)
|
||||
{
|
||||
if (r->len < STENO_FMT_MAX_OUTPUT - 1) {
|
||||
r->text[r->len++] = c;
|
||||
}
|
||||
}
|
||||
|
||||
/* ── apply capitalization/mode transforms to a text segment ───── */
|
||||
|
||||
static void emit_text_transformed(struct steno_fmt_state *state,
|
||||
struct steno_fmt_result *r,
|
||||
const char *text, size_t len)
|
||||
{
|
||||
if (len == 0) return;
|
||||
|
||||
for (size_t i = 0; i < len; i++) {
|
||||
char c = text[i];
|
||||
|
||||
/* Mode transforms */
|
||||
if (state->mode == STENO_MODE_CAPS) {
|
||||
c = to_upper(c);
|
||||
} else if (state->mode == STENO_MODE_LOWER) {
|
||||
c = to_lower(c);
|
||||
} else if (state->mode == STENO_MODE_TITLE) {
|
||||
/* Capitalize first letter of each word */
|
||||
if (i == 0 || (i > 0 && text[i - 1] == ' ')) {
|
||||
c = to_upper(c);
|
||||
}
|
||||
}
|
||||
|
||||
/* One-shot transforms (first char only) */
|
||||
if (i == 0) {
|
||||
if (state->cap_next) {
|
||||
c = to_upper(c);
|
||||
state->cap_next = false;
|
||||
}
|
||||
if (state->upper_next) {
|
||||
/* uppercase entire word — handled below */
|
||||
}
|
||||
if (state->lower_next) {
|
||||
/* lowercase entire word — handled below */
|
||||
}
|
||||
}
|
||||
|
||||
/* upper_next: entire word */
|
||||
if (state->upper_next) {
|
||||
c = to_upper(c);
|
||||
}
|
||||
|
||||
/* lower_next: entire word */
|
||||
if (state->lower_next) {
|
||||
c = to_lower(c);
|
||||
}
|
||||
|
||||
emit_char(r, c);
|
||||
}
|
||||
|
||||
state->upper_next = false;
|
||||
state->lower_next = false;
|
||||
}
|
||||
|
||||
/* ── prepend space if needed ────────────────────────────────────── */
|
||||
|
||||
static void maybe_space(struct steno_fmt_state *state,
|
||||
struct steno_fmt_result *r)
|
||||
{
|
||||
if (state->space_pending && !state->suppress_space) {
|
||||
emit_char(r, ' ');
|
||||
}
|
||||
state->suppress_space = false;
|
||||
}
|
||||
|
||||
/* ── process a single {…} command ──────────────────────────────── */
|
||||
|
||||
static void process_command(struct steno_fmt_state *state,
|
||||
struct steno_fmt_result *r,
|
||||
const char *cmd, size_t cmd_len)
|
||||
{
|
||||
/* {^} — suppress space */
|
||||
if (cmd_len == 1 && cmd[0] == '^') {
|
||||
state->suppress_space = true;
|
||||
return;
|
||||
}
|
||||
|
||||
/* {^suffix} — attach suffix */
|
||||
if (cmd_len > 1 && cmd[0] == '^') {
|
||||
state->suppress_space = true;
|
||||
maybe_space(state, r);
|
||||
emit_text_transformed(state, r, cmd + 1, cmd_len - 1);
|
||||
state->space_pending = true;
|
||||
state->glue = false;
|
||||
return;
|
||||
}
|
||||
|
||||
/* {prefix^} — attach prefix */
|
||||
if (cmd_len > 1 && cmd[cmd_len - 1] == '^') {
|
||||
maybe_space(state, r);
|
||||
emit_text_transformed(state, r, cmd, cmd_len - 1);
|
||||
state->suppress_space = true;
|
||||
state->space_pending = false;
|
||||
state->glue = false;
|
||||
return;
|
||||
}
|
||||
|
||||
/* {-|} — capitalize next */
|
||||
if (cmd_len == 2 && cmd[0] == '-' && cmd[1] == '|') {
|
||||
state->cap_next = true;
|
||||
return;
|
||||
}
|
||||
|
||||
/* {~|} — carry capitalize */
|
||||
if (cmd_len == 2 && cmd[0] == '~' && cmd[1] == '|') {
|
||||
state->cap_next = true;
|
||||
return;
|
||||
}
|
||||
|
||||
/* {*-|} — retro capitalize (backspace + re-emit) */
|
||||
if (cmd_len == 3 && cmd[0] == '*' && cmd[1] == '-' && cmd[2] == '|') {
|
||||
/* Signal retro-capitalize — simplified: set flag, caller handles */
|
||||
r->backspaces = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
/* {*!} — retro delete space */
|
||||
if (cmd_len == 2 && cmd[0] == '*' && cmd[1] == '!') {
|
||||
r->backspaces = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
/* {*?} — retro insert space */
|
||||
if (cmd_len == 2 && cmd[0] == '*' && cmd[1] == '?') {
|
||||
/* Would need undo context; simplified signal */
|
||||
return;
|
||||
}
|
||||
|
||||
/* {*} — undo */
|
||||
if (cmd_len == 1 && cmd[0] == '*') {
|
||||
r->is_undo = true;
|
||||
r->is_command_only = true;
|
||||
return;
|
||||
}
|
||||
|
||||
/* {.} {,} {?} {!} {;} {:} — punctuation: attach to prev, cap next for sentence-enders */
|
||||
if (cmd_len == 1 && (cmd[0] == '.' || cmd[0] == ',' ||
|
||||
cmd[0] == '?' || cmd[0] == '!' ||
|
||||
cmd[0] == ';' || cmd[0] == ':')) {
|
||||
state->suppress_space = true;
|
||||
maybe_space(state, r);
|
||||
emit_char(r, cmd[0]);
|
||||
state->space_pending = true;
|
||||
state->suppress_space = false;
|
||||
/* Sentence-ending punctuation capitalizes next */
|
||||
if (cmd[0] == '.' || cmd[0] == '?' || cmd[0] == '!') {
|
||||
state->cap_next = true;
|
||||
}
|
||||
state->glue = false;
|
||||
return;
|
||||
}
|
||||
|
||||
/* {#...} — key combo (emit nothing for now, could extend) */
|
||||
if (cmd_len > 0 && cmd[0] == '#') {
|
||||
r->is_command_only = true;
|
||||
return;
|
||||
}
|
||||
|
||||
/* {&letter} — fingerspelling */
|
||||
if (cmd_len >= 2 && cmd[0] == '&') {
|
||||
/* Glue: no space between glue strokes */
|
||||
if (state->glue) {
|
||||
state->suppress_space = true;
|
||||
}
|
||||
maybe_space(state, r);
|
||||
emit_text_transformed(state, r, cmd + 1, cmd_len - 1);
|
||||
state->space_pending = true;
|
||||
state->glue = true;
|
||||
state->suppress_space = false;
|
||||
return;
|
||||
}
|
||||
|
||||
/* {&} — bare glue */
|
||||
if (cmd_len == 1 && cmd[0] == '&') {
|
||||
state->glue = true;
|
||||
state->suppress_space = true;
|
||||
return;
|
||||
}
|
||||
|
||||
/* {MODE:CAPS} {MODE:TITLE} {MODE:LOWER} {MODE:RESET} */
|
||||
if (starts_with(cmd, cmd_len, "MODE:")) {
|
||||
const char *mode_str = cmd + 5;
|
||||
size_t mode_len = cmd_len - 5;
|
||||
if (mode_len == 4 && str_eq(mode_str, "CAPS", 4)) {
|
||||
state->mode = STENO_MODE_CAPS;
|
||||
} else if (mode_len == 5 && str_eq(mode_str, "TITLE", 5)) {
|
||||
state->mode = STENO_MODE_TITLE;
|
||||
} else if (mode_len == 5 && str_eq(mode_str, "LOWER", 5)) {
|
||||
state->mode = STENO_MODE_LOWER;
|
||||
} else if (mode_len == 5 && str_eq(mode_str, "RESET", 5)) {
|
||||
state->mode = STENO_MODE_NORMAL;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
/* {<} — uppercase next word */
|
||||
if (cmd_len == 1 && cmd[0] == '<') {
|
||||
state->upper_next = true;
|
||||
return;
|
||||
}
|
||||
|
||||
/* {>} — lowercase next word */
|
||||
if (cmd_len == 1 && cmd[0] == '>') {
|
||||
state->lower_next = true;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/* ── main entry point ──────────────────────────────────────────── */
|
||||
|
||||
void steno_fmt_init(struct steno_fmt_state *state)
|
||||
{
|
||||
memset(state, 0, sizeof(*state));
|
||||
state->space_pending = false;
|
||||
state->cap_next = false;
|
||||
state->upper_next = false;
|
||||
state->lower_next = false;
|
||||
state->suppress_space = false;
|
||||
state->glue = false;
|
||||
state->mode = STENO_MODE_NORMAL;
|
||||
}
|
||||
|
||||
void steno_fmt_process(struct steno_fmt_state *state,
|
||||
const char *translation,
|
||||
struct steno_fmt_result *result)
|
||||
{
|
||||
memset(result, 0, sizeof(*result));
|
||||
|
||||
if (!translation || !translation[0]) {
|
||||
result->is_command_only = true;
|
||||
return;
|
||||
}
|
||||
|
||||
size_t tlen = strlen(translation);
|
||||
const char *p = translation;
|
||||
const char *end = translation + tlen;
|
||||
bool emitted_text = false;
|
||||
bool had_command = false;
|
||||
bool this_glue = false;
|
||||
|
||||
while (p < end) {
|
||||
/* Escaped braces: \{ \} */
|
||||
if (p[0] == '\\' && p + 1 < end && (p[1] == '{' || p[1] == '}')) {
|
||||
maybe_space(state, result);
|
||||
emit_text_transformed(state, result, p + 1, 1);
|
||||
state->space_pending = true;
|
||||
state->glue = false;
|
||||
emitted_text = true;
|
||||
p += 2;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Command: {…} */
|
||||
if (p[0] == '{') {
|
||||
const char *close = p + 1;
|
||||
while (close < end && *close != '}') {
|
||||
close++;
|
||||
}
|
||||
if (close < end) {
|
||||
const char *cmd = p + 1;
|
||||
size_t cmd_len = (size_t)(close - cmd);
|
||||
process_command(state, result, cmd, cmd_len);
|
||||
had_command = true;
|
||||
if (state->glue) this_glue = true;
|
||||
p = close + 1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
/* Literal text segment — find end (next '{' or '\' or end) */
|
||||
const char *seg_start = p;
|
||||
while (p < end && p[0] != '{' &&
|
||||
!(p[0] == '\\' && p + 1 < end && (p[1] == '{' || p[1] == '}'))) {
|
||||
p++;
|
||||
}
|
||||
size_t seg_len = (size_t)(p - seg_start);
|
||||
if (seg_len > 0) {
|
||||
if (!this_glue) {
|
||||
maybe_space(state, result);
|
||||
}
|
||||
emit_text_transformed(state, result, seg_start, seg_len);
|
||||
state->space_pending = true;
|
||||
if (!this_glue) {
|
||||
state->glue = false;
|
||||
}
|
||||
emitted_text = true;
|
||||
}
|
||||
}
|
||||
|
||||
result->text[result->len] = '\0';
|
||||
result->is_command_only = !emitted_text && had_command && result->len == 0;
|
||||
}
|
||||
46
src/formatter.h
Normal file
46
src/formatter.h
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
#ifndef STENO_FORMATTER_H
|
||||
#define STENO_FORMATTER_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
/* Max output length from a single translation */
|
||||
#define STENO_FMT_MAX_OUTPUT 128
|
||||
|
||||
/* Formatting mode */
|
||||
enum steno_fmt_mode {
|
||||
STENO_MODE_NORMAL = 0,
|
||||
STENO_MODE_CAPS,
|
||||
STENO_MODE_TITLE,
|
||||
STENO_MODE_LOWER,
|
||||
};
|
||||
|
||||
/* State persists between translations */
|
||||
struct steno_fmt_state {
|
||||
bool space_pending; /* insert space before next word */
|
||||
bool cap_next; /* capitalize next word */
|
||||
bool upper_next; /* uppercase next word */
|
||||
bool lower_next; /* lowercase next word */
|
||||
bool suppress_space; /* {^} suppress next space */
|
||||
bool glue; /* in fingerspelling/glue sequence */
|
||||
enum steno_fmt_mode mode;
|
||||
};
|
||||
|
||||
/* Result of formatting one translation */
|
||||
struct steno_fmt_result {
|
||||
char text[STENO_FMT_MAX_OUTPUT]; /* output text to emit */
|
||||
uint8_t len; /* length of text */
|
||||
uint8_t backspaces; /* backspaces to send BEFORE text */
|
||||
bool is_command_only; /* true if no text output (pure command) */
|
||||
bool is_undo; /* true if this is an undo command */
|
||||
};
|
||||
|
||||
void steno_fmt_init(struct steno_fmt_state *state);
|
||||
|
||||
/* Format a raw translation string. Updates state, fills result. */
|
||||
void steno_fmt_process(struct steno_fmt_state *state,
|
||||
const char *translation,
|
||||
struct steno_fmt_result *result);
|
||||
|
||||
#endif
|
||||
352
src/output.c
Normal file
352
src/output.c
Normal file
|
|
@ -0,0 +1,352 @@
|
|||
#include "output.h"
|
||||
|
||||
#include <zephyr/kernel.h>
|
||||
#include <zephyr/logging/log.h>
|
||||
|
||||
#include <zmk/hid.h>
|
||||
#include <zmk/endpoints.h>
|
||||
#include <zmk/events/keycode_state_changed.h>
|
||||
|
||||
LOG_MODULE_DECLARE(zmk, CONFIG_ZMK_LOG_LEVEL);
|
||||
|
||||
struct hid_map {
|
||||
uint8_t keycode;
|
||||
bool shift;
|
||||
};
|
||||
|
||||
static const struct hid_map ASCII_TO_HID[128] = {
|
||||
[' '] = {0x2C, false}, /* space */
|
||||
['!'] = {0x1E, true}, /* shift+1 */
|
||||
['"'] = {0x34, true}, /* shift+' */
|
||||
['#'] = {0x20, true},
|
||||
['$'] = {0x21, true},
|
||||
['%'] = {0x22, true},
|
||||
['&'] = {0x24, true},
|
||||
['\''] = {0x34, false},
|
||||
['('] = {0x26, true},
|
||||
[')'] = {0x27, true},
|
||||
['*'] = {0x25, true},
|
||||
['+'] = {0x2E, true},
|
||||
[','] = {0x36, false},
|
||||
['-'] = {0x2D, false},
|
||||
['.'] = {0x37, false},
|
||||
['/'] = {0x38, false},
|
||||
['0'] = {0x27, false},
|
||||
['1'] = {0x1E, false},
|
||||
['2'] = {0x1F, false},
|
||||
['3'] = {0x20, false},
|
||||
['4'] = {0x21, false},
|
||||
['5'] = {0x22, false},
|
||||
['6'] = {0x23, false},
|
||||
['7'] = {0x24, false},
|
||||
['8'] = {0x25, false},
|
||||
['9'] = {0x26, false},
|
||||
[':'] = {0x33, true},
|
||||
[';'] = {0x33, false},
|
||||
['<'] = {0x36, true},
|
||||
['='] = {0x2E, false},
|
||||
['>'] = {0x37, true},
|
||||
['?'] = {0x38, true},
|
||||
['@'] = {0x1F, true},
|
||||
['A'] = {0x04, true},
|
||||
['B'] = {0x05, true},
|
||||
['C'] = {0x06, true},
|
||||
['D'] = {0x07, true},
|
||||
['E'] = {0x08, true},
|
||||
['F'] = {0x09, true},
|
||||
['G'] = {0x0A, true},
|
||||
['H'] = {0x0B, true},
|
||||
['I'] = {0x0C, true},
|
||||
['J'] = {0x0D, true},
|
||||
['K'] = {0x0E, true},
|
||||
['L'] = {0x0F, true},
|
||||
['M'] = {0x10, true},
|
||||
['N'] = {0x11, true},
|
||||
['O'] = {0x12, true},
|
||||
['P'] = {0x13, true},
|
||||
['Q'] = {0x14, true},
|
||||
['R'] = {0x15, true},
|
||||
['S'] = {0x16, true},
|
||||
['T'] = {0x17, true},
|
||||
['U'] = {0x18, true},
|
||||
['V'] = {0x19, true},
|
||||
['W'] = {0x1A, true},
|
||||
['X'] = {0x1B, true},
|
||||
['Y'] = {0x1C, true},
|
||||
['Z'] = {0x1D, true},
|
||||
['['] = {0x2F, false},
|
||||
['\\'] = {0x31, false},
|
||||
[']'] = {0x30, false},
|
||||
['^'] = {0x23, true},
|
||||
['_'] = {0x2D, true},
|
||||
['`'] = {0x35, false},
|
||||
['a'] = {0x04, false},
|
||||
['b'] = {0x05, false},
|
||||
['c'] = {0x06, false},
|
||||
['d'] = {0x07, false},
|
||||
['e'] = {0x08, false},
|
||||
['f'] = {0x09, false},
|
||||
['g'] = {0x0A, false},
|
||||
['h'] = {0x0B, false},
|
||||
['i'] = {0x0C, false},
|
||||
['j'] = {0x0D, false},
|
||||
['k'] = {0x0E, false},
|
||||
['l'] = {0x0F, false},
|
||||
['m'] = {0x10, false},
|
||||
['n'] = {0x11, false},
|
||||
['o'] = {0x12, false},
|
||||
['p'] = {0x13, false},
|
||||
['q'] = {0x14, false},
|
||||
['r'] = {0x15, false},
|
||||
['s'] = {0x16, false},
|
||||
['t'] = {0x17, false},
|
||||
['u'] = {0x18, false},
|
||||
['v'] = {0x19, false},
|
||||
['w'] = {0x1A, false},
|
||||
['x'] = {0x1B, false},
|
||||
['y'] = {0x1C, false},
|
||||
['z'] = {0x1D, false},
|
||||
['{'] = {0x2F, true},
|
||||
['|'] = {0x31, true},
|
||||
['}'] = {0x30, true},
|
||||
['~'] = {0x35, true},
|
||||
};
|
||||
|
||||
#define HID_BACKSPACE 0x2A
|
||||
#define HID_RETURN 0x28
|
||||
#define HID_LSHIFT 0xE1
|
||||
#define HID_LCTRL 0xE0
|
||||
#define HID_LALT 0xE2
|
||||
#define HID_RALT 0xE6
|
||||
|
||||
static void tap_key(uint8_t keycode, bool shift)
|
||||
{
|
||||
if (shift) {
|
||||
raise_zmk_keycode_state_changed((struct zmk_keycode_state_changed){
|
||||
.usage_page = 0x07, .keycode = HID_LSHIFT,
|
||||
.implicit_modifiers = 0, .explicit_modifiers = 0,
|
||||
.state = true, .timestamp = k_uptime_get()});
|
||||
}
|
||||
|
||||
raise_zmk_keycode_state_changed((struct zmk_keycode_state_changed){
|
||||
.usage_page = 0x07, .keycode = keycode,
|
||||
.implicit_modifiers = 0, .explicit_modifiers = 0,
|
||||
.state = true, .timestamp = k_uptime_get()});
|
||||
|
||||
raise_zmk_keycode_state_changed((struct zmk_keycode_state_changed){
|
||||
.usage_page = 0x07, .keycode = keycode,
|
||||
.implicit_modifiers = 0, .explicit_modifiers = 0,
|
||||
.state = false, .timestamp = k_uptime_get()});
|
||||
|
||||
if (shift) {
|
||||
raise_zmk_keycode_state_changed((struct zmk_keycode_state_changed){
|
||||
.usage_page = 0x07, .keycode = HID_LSHIFT,
|
||||
.implicit_modifiers = 0, .explicit_modifiers = 0,
|
||||
.state = false, .timestamp = k_uptime_get()});
|
||||
}
|
||||
}
|
||||
|
||||
static void press_key(uint8_t keycode)
|
||||
{
|
||||
raise_zmk_keycode_state_changed((struct zmk_keycode_state_changed){
|
||||
.usage_page = 0x07, .keycode = keycode,
|
||||
.implicit_modifiers = 0, .explicit_modifiers = 0,
|
||||
.state = true, .timestamp = k_uptime_get()});
|
||||
}
|
||||
|
||||
static void release_key(uint8_t keycode)
|
||||
{
|
||||
raise_zmk_keycode_state_changed((struct zmk_keycode_state_changed){
|
||||
.usage_page = 0x07, .keycode = keycode,
|
||||
.implicit_modifiers = 0, .explicit_modifiers = 0,
|
||||
.state = false, .timestamp = k_uptime_get()});
|
||||
}
|
||||
|
||||
/* Map hex digit (0-15) to HID keycode */
|
||||
static uint8_t hex_to_hid(uint8_t nib)
|
||||
{
|
||||
if (nib == 0) {
|
||||
return 0x27; /* '0' */
|
||||
}
|
||||
if (nib <= 9) {
|
||||
return 0x1E + (nib - 1); /* '1'-'9': 0x1E-0x26 */
|
||||
}
|
||||
return 0x04 + (nib - 10); /* 'a'-'f': 0x04-0x09 */
|
||||
}
|
||||
|
||||
/* Tap hex digits of codepoint (variable width, skip leading zeros) */
|
||||
static void tap_hex_digits(uint32_t codepoint)
|
||||
{
|
||||
char buf[9];
|
||||
int n = 0;
|
||||
|
||||
if (codepoint == 0) {
|
||||
tap_key(hex_to_hid(0), false);
|
||||
return;
|
||||
}
|
||||
|
||||
/* Build hex digits in reverse */
|
||||
uint32_t cp = codepoint;
|
||||
while (cp > 0) {
|
||||
buf[n++] = cp & 0xF;
|
||||
cp >>= 4;
|
||||
}
|
||||
|
||||
/* Tap in forward order */
|
||||
for (int i = n - 1; i >= 0; i--) {
|
||||
tap_key(hex_to_hid(buf[i]), false);
|
||||
}
|
||||
}
|
||||
|
||||
void steno_output_unicode(uint32_t codepoint)
|
||||
{
|
||||
if (codepoint > 0x10FFFF) {
|
||||
LOG_WRN("Invalid codepoint U+%06X", codepoint);
|
||||
return;
|
||||
}
|
||||
|
||||
#if IS_ENABLED(CONFIG_STENO_UNICODE_MODE_LINUX)
|
||||
/* IBus/GTK: Ctrl+Shift+U, hex digits, Return */
|
||||
press_key(HID_LCTRL);
|
||||
press_key(HID_LSHIFT);
|
||||
tap_key(0x18, false); /* 'u' */
|
||||
release_key(HID_LSHIFT);
|
||||
release_key(HID_LCTRL);
|
||||
tap_hex_digits(codepoint);
|
||||
tap_key(HID_RETURN, false);
|
||||
|
||||
#elif IS_ENABLED(CONFIG_STENO_UNICODE_MODE_MACOS)
|
||||
/* macOS Unicode Hex Input: hold Option, type 4+ hex digits */
|
||||
press_key(HID_LALT);
|
||||
|
||||
/* Pad to at least 4 digits */
|
||||
char digits[8];
|
||||
int n = 0;
|
||||
uint32_t cp = codepoint;
|
||||
|
||||
do {
|
||||
digits[n++] = cp & 0xF;
|
||||
cp >>= 4;
|
||||
} while (cp > 0);
|
||||
|
||||
/* Pad to 4 */
|
||||
while (n < 4) {
|
||||
digits[n++] = 0;
|
||||
}
|
||||
|
||||
for (int i = n - 1; i >= 0; i--) {
|
||||
tap_key(hex_to_hid(digits[i]), false);
|
||||
}
|
||||
|
||||
release_key(HID_LALT);
|
||||
|
||||
#elif IS_ENABLED(CONFIG_STENO_UNICODE_MODE_WINC)
|
||||
/* WinCompose: tap RAlt, 'u', hex digits, Return */
|
||||
tap_key(HID_RALT, false);
|
||||
tap_key(0x18, false); /* 'u' */
|
||||
tap_hex_digits(codepoint);
|
||||
tap_key(HID_RETURN, false);
|
||||
|
||||
#else
|
||||
LOG_WRN("Unicode disabled, skipping U+%04X", codepoint);
|
||||
#endif
|
||||
}
|
||||
|
||||
/* Decode UTF-8 byte at text[i], write codepoint, return bytes consumed (0 on error) */
|
||||
static int utf8_decode(const char *text, size_t len, size_t i, uint32_t *cp)
|
||||
{
|
||||
unsigned char c = (unsigned char)text[i];
|
||||
|
||||
if (c < 0x80) {
|
||||
*cp = c;
|
||||
return 1;
|
||||
}
|
||||
|
||||
uint32_t codepoint;
|
||||
int expect; /* expected continuation bytes */
|
||||
|
||||
if ((c & 0xE0) == 0xC0) {
|
||||
codepoint = c & 0x1F;
|
||||
expect = 1;
|
||||
} else if ((c & 0xF0) == 0xE0) {
|
||||
codepoint = c & 0x0F;
|
||||
expect = 2;
|
||||
} else if ((c & 0xF8) == 0xF0) {
|
||||
codepoint = c & 0x07;
|
||||
expect = 3;
|
||||
} else {
|
||||
return 0; /* invalid lead byte */
|
||||
}
|
||||
|
||||
if (i + expect >= len) {
|
||||
return 0; /* truncated */
|
||||
}
|
||||
|
||||
for (int j = 1; j <= expect; j++) {
|
||||
unsigned char cont = (unsigned char)text[i + j];
|
||||
if ((cont & 0xC0) != 0x80) {
|
||||
return 0;
|
||||
}
|
||||
codepoint = (codepoint << 6) | (cont & 0x3F);
|
||||
}
|
||||
|
||||
/* Reject overlong encodings */
|
||||
if ((expect == 1 && codepoint < 0x80) ||
|
||||
(expect == 2 && codepoint < 0x800) ||
|
||||
(expect == 3 && codepoint < 0x10000)) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Reject surrogates (U+D800..U+DFFF) and beyond Unicode max */
|
||||
if ((codepoint >= 0xD800 && codepoint <= 0xDFFF) || codepoint > 0x10FFFF) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
*cp = codepoint;
|
||||
return 1 + expect;
|
||||
}
|
||||
|
||||
void steno_output_send(const char *text, size_t len)
|
||||
{
|
||||
for (size_t i = 0; i < len; ) {
|
||||
unsigned char c = (unsigned char)text[i];
|
||||
|
||||
if (c == '\n') {
|
||||
tap_key(HID_RETURN, false);
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* ASCII range */
|
||||
if (c < 128) {
|
||||
if (ASCII_TO_HID[c].keycode == 0) {
|
||||
LOG_WRN("Unmapped ASCII char 0x%02X", c);
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
tap_key(ASCII_TO_HID[c].keycode, ASCII_TO_HID[c].shift);
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
/* Multi-byte UTF-8 → Unicode codepoint */
|
||||
uint32_t codepoint;
|
||||
int consumed = utf8_decode(text, len, i, &codepoint);
|
||||
|
||||
if (consumed == 0) {
|
||||
LOG_WRN("Invalid UTF-8 at offset %u (0x%02X)", (unsigned)i, c);
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
|
||||
steno_output_unicode(codepoint);
|
||||
i += consumed;
|
||||
}
|
||||
}
|
||||
|
||||
void steno_output_backspace(int count)
|
||||
{
|
||||
for (int i = 0; i < count; i++) {
|
||||
tap_key(HID_BACKSPACE, false);
|
||||
}
|
||||
}
|
||||
13
src/output.h
Normal file
13
src/output.h
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
#ifndef STENO_OUTPUT_H
|
||||
#define STENO_OUTPUT_H
|
||||
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
void steno_output_send(const char *text, size_t len);
|
||||
|
||||
void steno_output_backspace(int count);
|
||||
|
||||
void steno_output_unicode(uint32_t codepoint);
|
||||
|
||||
#endif
|
||||
148
src/trie.c
Normal file
148
src/trie.c
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
#include "trie.h"
|
||||
#include <string.h>
|
||||
|
||||
static const uint8_t *dict_data;
|
||||
static const struct steno_dict_header *dict_hdr;
|
||||
static const uint8_t *entry_array;
|
||||
static const char *string_table;
|
||||
static size_t entry_stride;
|
||||
|
||||
int steno_trie_init(const uint8_t *data, size_t len)
|
||||
{
|
||||
if (!data || len < sizeof(struct steno_dict_header)) {
|
||||
return -1;
|
||||
}
|
||||
|
||||
dict_hdr = (const struct steno_dict_header *)data;
|
||||
|
||||
if (dict_hdr->magic != STENO_DICT_MAGIC) {
|
||||
return -2;
|
||||
}
|
||||
if (dict_hdr->version != 1) {
|
||||
return -3;
|
||||
}
|
||||
if (dict_hdr->max_strokes == 0 || dict_hdr->max_strokes > 16) {
|
||||
return -4;
|
||||
}
|
||||
|
||||
entry_stride = (size_t)dict_hdr->max_strokes * 4 + 4;
|
||||
size_t entries_end = sizeof(struct steno_dict_header) +
|
||||
(size_t)dict_hdr->entry_count * entry_stride;
|
||||
|
||||
if (entries_end > len || dict_hdr->strings_offset > len) {
|
||||
return -5;
|
||||
}
|
||||
|
||||
dict_data = data;
|
||||
entry_array = data + sizeof(struct steno_dict_header);
|
||||
string_table = (const char *)(data + dict_hdr->strings_offset);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static const uint8_t *get_entry(uint32_t idx)
|
||||
{
|
||||
return entry_array + (size_t)idx * entry_stride;
|
||||
}
|
||||
|
||||
static int cmp_strokes(const uint32_t *a, uint8_t a_count,
|
||||
const uint8_t *entry_bytes, uint8_t max_s)
|
||||
{
|
||||
const uint32_t *b = (const uint32_t *)entry_bytes;
|
||||
|
||||
uint8_t b_count = 0;
|
||||
for (uint8_t i = 0; i < max_s; i++) {
|
||||
if (b[i] != 0) {
|
||||
b_count = i + 1;
|
||||
}
|
||||
}
|
||||
|
||||
uint8_t min_count = a_count < b_count ? a_count : b_count;
|
||||
for (uint8_t i = 0; i < min_count; i++) {
|
||||
if (a[i] < b[i]) return -1;
|
||||
if (a[i] > b[i]) return 1;
|
||||
}
|
||||
if (a_count < b_count) return -1;
|
||||
if (a_count > b_count) return 1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
const char *steno_trie_lookup(const uint32_t *strokes, uint8_t count)
|
||||
{
|
||||
if (!dict_hdr || !strokes || count == 0 || count > dict_hdr->max_strokes) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
uint32_t lo = 0;
|
||||
uint32_t hi = dict_hdr->entry_count;
|
||||
|
||||
while (lo < hi) {
|
||||
uint32_t mid = lo + (hi - lo) / 2;
|
||||
const uint8_t *entry = get_entry(mid);
|
||||
int c = cmp_strokes(strokes, count, entry, dict_hdr->max_strokes);
|
||||
if (c == 0) {
|
||||
uint32_t str_off;
|
||||
memcpy(&str_off, entry + (size_t)dict_hdr->max_strokes * 4, 4);
|
||||
return string_table + str_off;
|
||||
}
|
||||
if (c < 0) {
|
||||
hi = mid;
|
||||
} else {
|
||||
lo = mid + 1;
|
||||
}
|
||||
}
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
bool steno_trie_has_prefix(const uint32_t *strokes, uint8_t count)
|
||||
{
|
||||
if (!dict_hdr || !strokes || count == 0 || count >= dict_hdr->max_strokes) {
|
||||
return false;
|
||||
}
|
||||
|
||||
uint32_t lo = 0;
|
||||
uint32_t hi = dict_hdr->entry_count;
|
||||
uint32_t first_ge = hi;
|
||||
|
||||
while (lo < hi) {
|
||||
uint32_t mid = lo + (hi - lo) / 2;
|
||||
const uint8_t *entry = get_entry(mid);
|
||||
const uint32_t *e_strokes = (const uint32_t *)entry;
|
||||
|
||||
int cmp = 0;
|
||||
for (uint8_t i = 0; i < count; i++) {
|
||||
if (e_strokes[i] < strokes[i]) { cmp = -1; break; }
|
||||
if (e_strokes[i] > strokes[i]) { cmp = 1; break; }
|
||||
}
|
||||
|
||||
if (cmp >= 0) {
|
||||
first_ge = mid;
|
||||
hi = mid;
|
||||
} else {
|
||||
lo = mid + 1;
|
||||
}
|
||||
}
|
||||
|
||||
for (uint32_t idx = first_ge; idx < dict_hdr->entry_count; idx++) {
|
||||
const uint8_t *entry = get_entry(idx);
|
||||
const uint32_t *e_strokes = (const uint32_t *)entry;
|
||||
|
||||
bool prefix_match = true;
|
||||
for (uint8_t i = 0; i < count; i++) {
|
||||
if (e_strokes[i] != strokes[i]) {
|
||||
prefix_match = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!prefix_match) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (e_strokes[count] != 0) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
25
src/trie.h
Normal file
25
src/trie.h
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
#ifndef STENO_TRIE_H
|
||||
#define STENO_TRIE_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stddef.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#define STENO_DICT_MAGIC 0x4F4E5453 /* "STNO" */
|
||||
|
||||
struct steno_dict_header {
|
||||
uint32_t magic;
|
||||
uint16_t version;
|
||||
uint8_t max_strokes;
|
||||
uint8_t _pad;
|
||||
uint32_t entry_count;
|
||||
uint32_t strings_offset;
|
||||
} __attribute__((packed));
|
||||
|
||||
int steno_trie_init(const uint8_t *data, size_t len);
|
||||
|
||||
const char *steno_trie_lookup(const uint32_t *strokes, uint8_t count);
|
||||
|
||||
bool steno_trie_has_prefix(const uint32_t *strokes, uint8_t count);
|
||||
|
||||
#endif
|
||||
64
src/undo.c
Normal file
64
src/undo.c
Normal file
|
|
@ -0,0 +1,64 @@
|
|||
/*
|
||||
* Copyright (c) 2024 Afiq Zudin Hadi
|
||||
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
|
||||
*/
|
||||
|
||||
#include "undo.h"
|
||||
#include <string.h>
|
||||
|
||||
void steno_undo_init(struct stroke_history *hist)
|
||||
{
|
||||
memset(hist, 0, sizeof(*hist));
|
||||
}
|
||||
|
||||
void steno_undo_push(struct stroke_history *hist,
|
||||
const uint32_t *strokes, uint8_t stroke_count,
|
||||
uint8_t output_len, uint8_t space_before,
|
||||
uint8_t fmt_flags)
|
||||
{
|
||||
struct stroke_history_entry *e = &hist->entries[hist->head];
|
||||
|
||||
uint8_t n = stroke_count;
|
||||
if (n > STENO_MAX_MULTI_STROKE) {
|
||||
n = STENO_MAX_MULTI_STROKE;
|
||||
}
|
||||
|
||||
memcpy(e->strokes, strokes, n * sizeof(uint32_t));
|
||||
e->stroke_count = n;
|
||||
e->output_len = output_len;
|
||||
e->space_before = space_before;
|
||||
e->fmt_flags = fmt_flags;
|
||||
|
||||
hist->head = (hist->head + 1) % CONFIG_STENO_HISTORY_SIZE;
|
||||
|
||||
if (hist->count < CONFIG_STENO_HISTORY_SIZE) {
|
||||
hist->count++;
|
||||
}
|
||||
}
|
||||
|
||||
struct stroke_history_entry *steno_undo_pop(struct stroke_history *hist)
|
||||
{
|
||||
if (hist->count == 0) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
hist->head = (hist->head + CONFIG_STENO_HISTORY_SIZE - 1) % CONFIG_STENO_HISTORY_SIZE;
|
||||
hist->count--;
|
||||
|
||||
return &hist->entries[hist->head];
|
||||
}
|
||||
|
||||
const struct stroke_history_entry *steno_undo_peek(const struct stroke_history *hist)
|
||||
{
|
||||
if (hist->count == 0) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
uint16_t idx = (hist->head + CONFIG_STENO_HISTORY_SIZE - 1) % CONFIG_STENO_HISTORY_SIZE;
|
||||
return &hist->entries[idx];
|
||||
}
|
||||
|
||||
uint16_t steno_undo_count(const struct stroke_history *hist)
|
||||
{
|
||||
return hist->count;
|
||||
}
|
||||
44
src/undo.h
Normal file
44
src/undo.h
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
#ifndef STENO_UNDO_H
|
||||
#define STENO_UNDO_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#ifndef CONFIG_STENO_HISTORY_SIZE
|
||||
#define CONFIG_STENO_HISTORY_SIZE 100
|
||||
#endif
|
||||
|
||||
#define STENO_MAX_MULTI_STROKE 8
|
||||
|
||||
struct stroke_history_entry {
|
||||
uint32_t strokes[STENO_MAX_MULTI_STROKE];
|
||||
uint8_t stroke_count;
|
||||
uint8_t output_len; /* chars emitted (for backspace count) */
|
||||
uint8_t space_before; /* 1 if space was prepended */
|
||||
uint8_t fmt_flags; /* formatter state snapshot for restore */
|
||||
};
|
||||
|
||||
struct stroke_history {
|
||||
struct stroke_history_entry entries[CONFIG_STENO_HISTORY_SIZE];
|
||||
uint16_t head; /* next write position */
|
||||
uint16_t count; /* entries in buffer */
|
||||
};
|
||||
|
||||
void steno_undo_init(struct stroke_history *hist);
|
||||
|
||||
/* Push a new entry after successful output */
|
||||
void steno_undo_push(struct stroke_history *hist,
|
||||
const uint32_t *strokes, uint8_t stroke_count,
|
||||
uint8_t output_len, uint8_t space_before,
|
||||
uint8_t fmt_flags);
|
||||
|
||||
/* Pop most recent entry for undo. Returns NULL if empty. */
|
||||
struct stroke_history_entry *steno_undo_pop(struct stroke_history *hist);
|
||||
|
||||
/* Peek at most recent entry without removing. Returns NULL if empty. */
|
||||
const struct stroke_history_entry *steno_undo_peek(const struct stroke_history *hist);
|
||||
|
||||
/* Get current count */
|
||||
uint16_t steno_undo_count(const struct stroke_history *hist);
|
||||
|
||||
#endif
|
||||
64
tests/run_tests.sh
Executable file
64
tests/run_tests.sh
Executable file
|
|
@ -0,0 +1,64 @@
|
|||
#!/bin/bash
|
||||
# Run MPHF dictionary tests: compile test dict, build C test, execute.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
BUILD_DIR="$SCRIPT_DIR/build"
|
||||
|
||||
mkdir -p "$BUILD_DIR"
|
||||
|
||||
echo "=== Step 1: Create test dictionary JSON ==="
|
||||
cat > "$BUILD_DIR/test_dict.json" << 'DICTEOF'
|
||||
{
|
||||
"S": "is",
|
||||
"T": "it",
|
||||
"THE": "the",
|
||||
"KAT": "cat",
|
||||
"TK": "did",
|
||||
"SKP": "and",
|
||||
"TPOR": "for",
|
||||
"STO": "so",
|
||||
"HAOEU": "hi",
|
||||
"TKOGS": "dogs",
|
||||
"S/T": "{.}",
|
||||
"PHAO*EUP/HRAOEUPB": "my line"
|
||||
}
|
||||
DICTEOF
|
||||
|
||||
echo "=== Step 2: Compile test dictionary ==="
|
||||
python3 "$ROOT_DIR/tools/compile_mphf.py" \
|
||||
"$BUILD_DIR/test_dict.json" \
|
||||
"$BUILD_DIR/test_dict.bin" \
|
||||
--stats
|
||||
|
||||
echo ""
|
||||
echo "=== Step 3: Build C test binary ==="
|
||||
cc -O2 -Wall -Wextra -I"$ROOT_DIR/src" \
|
||||
-o "$BUILD_DIR/test_mphf" \
|
||||
"$SCRIPT_DIR/test_mphf.c" \
|
||||
"$ROOT_DIR/src/dict_mphf.c"
|
||||
|
||||
echo ""
|
||||
echo "=== Step 4: Run tests ==="
|
||||
"$BUILD_DIR/test_mphf" "$BUILD_DIR/test_dict.bin"
|
||||
|
||||
echo ""
|
||||
echo "=== Step 5: Run with larger dict (if Plover available) ==="
|
||||
PLOVER="/tmp/plover-main.json"
|
||||
if [ -f "$PLOVER" ]; then
|
||||
echo "Compiling 1000-entry subset..."
|
||||
python3 "$ROOT_DIR/tools/compile_mphf.py" \
|
||||
"$PLOVER" \
|
||||
"$BUILD_DIR/plover_1k.bin" \
|
||||
--max-entries 1000 \
|
||||
--stats
|
||||
echo ""
|
||||
echo "Binary size: $(wc -c < "$BUILD_DIR/plover_1k.bin") bytes"
|
||||
else
|
||||
echo "Plover dict not found at $PLOVER, skipping large dict test"
|
||||
echo "Download: curl -sL 'https://raw.githubusercontent.com/openstenoproject/plover/main/plover/assets/main.json' -o /tmp/plover-main.json"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Done ==="
|
||||
340
tests/test_formatter.c
Normal file
340
tests/test_formatter.c
Normal file
|
|
@ -0,0 +1,340 @@
|
|||
/*
|
||||
* Native host test for Plover formatting engine.
|
||||
* Build: cc -I../src -o test_formatter test_formatter.c ../src/formatter.c
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
#include "formatter.h"
|
||||
|
||||
static int tests_run = 0;
|
||||
static int tests_passed = 0;
|
||||
|
||||
#define ASSERT_EQ_STR(actual, expected, msg) do { \
|
||||
tests_run++; \
|
||||
if (strcmp((actual), (expected)) == 0) { \
|
||||
tests_passed++; \
|
||||
} else { \
|
||||
printf("FAIL [%s]: got \"%s\", expected \"%s\"\n", msg, actual, expected); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define ASSERT_EQ_INT(actual, expected, msg) do { \
|
||||
tests_run++; \
|
||||
if ((actual) == (expected)) { \
|
||||
tests_passed++; \
|
||||
} else { \
|
||||
printf("FAIL [%s]: got %d, expected %d\n", msg, (int)(actual), (int)(expected)); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
#define ASSERT_TRUE(cond, msg) do { \
|
||||
tests_run++; \
|
||||
if ((cond)) { \
|
||||
tests_passed++; \
|
||||
} else { \
|
||||
printf("FAIL [%s]\n", msg); \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
/* Helper: process and return output text */
|
||||
static struct steno_fmt_result proc(struct steno_fmt_state *s, const char *t)
|
||||
{
|
||||
struct steno_fmt_result r;
|
||||
steno_fmt_process(s, t, &r);
|
||||
return r;
|
||||
}
|
||||
|
||||
static void test_raw_text(void)
|
||||
{
|
||||
struct steno_fmt_state s;
|
||||
steno_fmt_init(&s);
|
||||
|
||||
/* First word: no space before */
|
||||
struct steno_fmt_result r = proc(&s, "hello");
|
||||
ASSERT_EQ_STR(r.text, "hello", "raw: first word");
|
||||
ASSERT_EQ_INT(r.len, 5, "raw: first word len");
|
||||
|
||||
/* Second word: space before */
|
||||
r = proc(&s, "world");
|
||||
ASSERT_EQ_STR(r.text, " world", "raw: second word with space");
|
||||
ASSERT_EQ_INT(r.len, 6, "raw: second word len");
|
||||
}
|
||||
|
||||
static void test_attach(void)
|
||||
{
|
||||
struct steno_fmt_state s;
|
||||
steno_fmt_init(&s);
|
||||
|
||||
proc(&s, "walk");
|
||||
struct steno_fmt_result r = proc(&s, "{^}");
|
||||
ASSERT_TRUE(r.is_command_only || r.len == 0, "attach: no output");
|
||||
|
||||
r = proc(&s, "ing");
|
||||
ASSERT_EQ_STR(r.text, "ing", "attach: no space after {^}");
|
||||
}
|
||||
|
||||
static void test_suffix_attach(void)
|
||||
{
|
||||
struct steno_fmt_state s;
|
||||
steno_fmt_init(&s);
|
||||
|
||||
proc(&s, "walk");
|
||||
struct steno_fmt_result r = proc(&s, "{^ing}");
|
||||
ASSERT_EQ_STR(r.text, "ing", "suffix: attached");
|
||||
ASSERT_EQ_INT(r.len, 3, "suffix: len");
|
||||
}
|
||||
|
||||
static void test_prefix_attach(void)
|
||||
{
|
||||
struct steno_fmt_state s;
|
||||
steno_fmt_init(&s);
|
||||
|
||||
struct steno_fmt_result r = proc(&s, "{pre^}");
|
||||
ASSERT_EQ_STR(r.text, "pre", "prefix: text");
|
||||
|
||||
r = proc(&s, "fix");
|
||||
ASSERT_EQ_STR(r.text, "fix", "prefix: next word attached");
|
||||
}
|
||||
|
||||
static void test_capitalize_next(void)
|
||||
{
|
||||
struct steno_fmt_state s;
|
||||
steno_fmt_init(&s);
|
||||
|
||||
proc(&s, "{-|}");
|
||||
struct steno_fmt_result r = proc(&s, "hello");
|
||||
ASSERT_EQ_STR(r.text, "Hello", "cap_next: first word capitalized");
|
||||
|
||||
/* Verify cap_next is one-shot */
|
||||
r = proc(&s, "world");
|
||||
ASSERT_EQ_STR(r.text, " world", "cap_next: one-shot reset");
|
||||
}
|
||||
|
||||
static void test_punctuation(void)
|
||||
{
|
||||
struct steno_fmt_state s;
|
||||
steno_fmt_init(&s);
|
||||
|
||||
proc(&s, "hello");
|
||||
struct steno_fmt_result r = proc(&s, "{.}");
|
||||
ASSERT_EQ_STR(r.text, ".", "period: attached");
|
||||
|
||||
/* Next word should be capitalized */
|
||||
r = proc(&s, "world");
|
||||
ASSERT_EQ_STR(r.text, " World", "period: cap next");
|
||||
}
|
||||
|
||||
static void test_comma(void)
|
||||
{
|
||||
struct steno_fmt_state s;
|
||||
steno_fmt_init(&s);
|
||||
|
||||
proc(&s, "hello");
|
||||
struct steno_fmt_result r = proc(&s, "{,}");
|
||||
ASSERT_EQ_STR(r.text, ",", "comma: attached");
|
||||
|
||||
/* Comma does NOT capitalize next */
|
||||
r = proc(&s, "world");
|
||||
ASSERT_EQ_STR(r.text, " world", "comma: no cap next");
|
||||
}
|
||||
|
||||
static void test_sentence_flow(void)
|
||||
{
|
||||
struct steno_fmt_state s;
|
||||
steno_fmt_init(&s);
|
||||
|
||||
struct steno_fmt_result r;
|
||||
r = proc(&s, "I");
|
||||
ASSERT_EQ_STR(r.text, "I", "sentence: I");
|
||||
|
||||
r = proc(&s, "{.}");
|
||||
ASSERT_EQ_STR(r.text, ".", "sentence: period");
|
||||
|
||||
r = proc(&s, "the");
|
||||
ASSERT_EQ_STR(r.text, " The", "sentence: The after period");
|
||||
}
|
||||
|
||||
static void test_mode_caps(void)
|
||||
{
|
||||
struct steno_fmt_state s;
|
||||
steno_fmt_init(&s);
|
||||
|
||||
proc(&s, "{MODE:CAPS}");
|
||||
struct steno_fmt_result r = proc(&s, "hello");
|
||||
ASSERT_EQ_STR(r.text, "HELLO", "mode_caps: uppercase");
|
||||
|
||||
r = proc(&s, "world");
|
||||
ASSERT_EQ_STR(r.text, " WORLD", "mode_caps: persists");
|
||||
|
||||
proc(&s, "{MODE:RESET}");
|
||||
r = proc(&s, "test");
|
||||
ASSERT_EQ_STR(r.text, " test", "mode_reset: normal");
|
||||
}
|
||||
|
||||
static void test_mode_title(void)
|
||||
{
|
||||
struct steno_fmt_state s;
|
||||
steno_fmt_init(&s);
|
||||
|
||||
proc(&s, "{MODE:TITLE}");
|
||||
struct steno_fmt_result r = proc(&s, "hello");
|
||||
ASSERT_EQ_STR(r.text, "Hello", "mode_title: capitalize");
|
||||
|
||||
proc(&s, "{MODE:RESET}");
|
||||
}
|
||||
|
||||
static void test_mode_lower(void)
|
||||
{
|
||||
struct steno_fmt_state s;
|
||||
steno_fmt_init(&s);
|
||||
|
||||
proc(&s, "{MODE:LOWER}");
|
||||
struct steno_fmt_result r = proc(&s, "HELLO");
|
||||
ASSERT_EQ_STR(r.text, "hello", "mode_lower: lowercase");
|
||||
|
||||
proc(&s, "{MODE:RESET}");
|
||||
}
|
||||
|
||||
static void test_fingerspelling(void)
|
||||
{
|
||||
struct steno_fmt_state s;
|
||||
steno_fmt_init(&s);
|
||||
|
||||
struct steno_fmt_result r;
|
||||
r = proc(&s, "{&a}");
|
||||
ASSERT_EQ_STR(r.text, "a", "finger: a");
|
||||
|
||||
r = proc(&s, "{&b}");
|
||||
ASSERT_EQ_STR(r.text, "b", "finger: b glued");
|
||||
|
||||
r = proc(&s, "{&c}");
|
||||
ASSERT_EQ_STR(r.text, "c", "finger: c glued");
|
||||
|
||||
/* Non-fingerspelling should get space */
|
||||
r = proc(&s, "hello");
|
||||
ASSERT_EQ_STR(r.text, " hello", "finger: break with space");
|
||||
}
|
||||
|
||||
static void test_combined_commands(void)
|
||||
{
|
||||
struct steno_fmt_state s;
|
||||
steno_fmt_init(&s);
|
||||
|
||||
proc(&s, "test");
|
||||
|
||||
/* {^}{-|} — suppress space + capitalize */
|
||||
struct steno_fmt_result r = proc(&s, "{^}{-|}");
|
||||
ASSERT_TRUE(r.len == 0 || r.is_command_only, "combined: no text");
|
||||
|
||||
r = proc(&s, "word");
|
||||
ASSERT_EQ_STR(r.text, "Word", "combined: attached + capitalized");
|
||||
}
|
||||
|
||||
static void test_undo(void)
|
||||
{
|
||||
struct steno_fmt_state s;
|
||||
steno_fmt_init(&s);
|
||||
|
||||
struct steno_fmt_result r = proc(&s, "{*}");
|
||||
ASSERT_TRUE(r.is_undo, "undo: flag set");
|
||||
ASSERT_TRUE(r.is_command_only, "undo: command only");
|
||||
}
|
||||
|
||||
static void test_key_combo(void)
|
||||
{
|
||||
struct steno_fmt_state s;
|
||||
steno_fmt_init(&s);
|
||||
|
||||
struct steno_fmt_result r = proc(&s, "{#Return}");
|
||||
ASSERT_TRUE(r.is_command_only, "key_combo: command only");
|
||||
}
|
||||
|
||||
static void test_literal_braces(void)
|
||||
{
|
||||
struct steno_fmt_state s;
|
||||
steno_fmt_init(&s);
|
||||
|
||||
struct steno_fmt_result r = proc(&s, "\\{");
|
||||
ASSERT_EQ_STR(r.text, "{", "literal: left brace");
|
||||
|
||||
r = proc(&s, "\\}");
|
||||
ASSERT_EQ_STR(r.text, " }", "literal: right brace with space");
|
||||
}
|
||||
|
||||
static void test_uppercase_next(void)
|
||||
{
|
||||
struct steno_fmt_state s;
|
||||
steno_fmt_init(&s);
|
||||
|
||||
proc(&s, "{<}");
|
||||
struct steno_fmt_result r = proc(&s, "hello");
|
||||
ASSERT_EQ_STR(r.text, "HELLO", "upper_next: entire word");
|
||||
|
||||
/* One-shot */
|
||||
r = proc(&s, "world");
|
||||
ASSERT_EQ_STR(r.text, " world", "upper_next: one-shot");
|
||||
}
|
||||
|
||||
static void test_lowercase_next(void)
|
||||
{
|
||||
struct steno_fmt_state s;
|
||||
steno_fmt_init(&s);
|
||||
|
||||
proc(&s, "{>}");
|
||||
struct steno_fmt_result r = proc(&s, "HELLO");
|
||||
ASSERT_EQ_STR(r.text, "hello", "lower_next: entire word");
|
||||
}
|
||||
|
||||
static void test_empty_translation(void)
|
||||
{
|
||||
struct steno_fmt_state s;
|
||||
steno_fmt_init(&s);
|
||||
|
||||
struct steno_fmt_result r = proc(&s, "");
|
||||
ASSERT_TRUE(r.is_command_only, "empty: command only");
|
||||
ASSERT_EQ_INT(r.len, 0, "empty: no text");
|
||||
}
|
||||
|
||||
static void test_prefix_then_suffix(void)
|
||||
{
|
||||
struct steno_fmt_state s;
|
||||
steno_fmt_init(&s);
|
||||
|
||||
/* {re^} then {^ed} */
|
||||
proc(&s, "{re^}");
|
||||
struct steno_fmt_result r = proc(&s, "{^ed}");
|
||||
ASSERT_EQ_STR(r.text, "ed", "pre+suf: attached");
|
||||
}
|
||||
|
||||
int main(void)
|
||||
{
|
||||
test_raw_text();
|
||||
test_attach();
|
||||
test_suffix_attach();
|
||||
test_prefix_attach();
|
||||
test_capitalize_next();
|
||||
test_punctuation();
|
||||
test_comma();
|
||||
test_sentence_flow();
|
||||
test_mode_caps();
|
||||
test_mode_title();
|
||||
test_mode_lower();
|
||||
test_fingerspelling();
|
||||
test_combined_commands();
|
||||
test_undo();
|
||||
test_key_combo();
|
||||
test_literal_braces();
|
||||
test_uppercase_next();
|
||||
test_lowercase_next();
|
||||
test_empty_translation();
|
||||
test_prefix_then_suffix();
|
||||
|
||||
printf("\n%d/%d tests passed\n", tests_passed, tests_run);
|
||||
if (tests_passed == tests_run) {
|
||||
printf("ALL TESTS PASSED\n");
|
||||
return 0;
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
383
tests/test_mphf.c
Normal file
383
tests/test_mphf.c
Normal file
|
|
@ -0,0 +1,383 @@
|
|||
/**
|
||||
* Native tests for MPHF dictionary engine.
|
||||
*
|
||||
* Builds and runs on host (not ZMK). Compiles a small test dictionary
|
||||
* via compile_mphf.py, then exercises all lookup paths in C.
|
||||
*
|
||||
* Build: cc -O2 -I../src -o test_mphf test_mphf.c ../src/dict_mphf.c
|
||||
* Run: ./test_mphf test_dict.bin
|
||||
*
|
||||
* SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <assert.h>
|
||||
#include "dict_mphf.h"
|
||||
|
||||
/* ─── Test helpers ─── */
|
||||
|
||||
static int tests_run = 0;
|
||||
static int tests_passed = 0;
|
||||
|
||||
#define TEST(name) \
|
||||
do { printf(" %-50s ", #name); tests_run++; } while(0)
|
||||
|
||||
#define PASS() \
|
||||
do { tests_passed++; printf("PASS\n"); } while(0)
|
||||
|
||||
#define FAIL(msg) \
|
||||
do { printf("FAIL: %s\n", msg); } while(0)
|
||||
|
||||
#define ASSERT_EQ_INT(a, b) \
|
||||
do { \
|
||||
if ((a) != (b)) { \
|
||||
char _buf[128]; \
|
||||
snprintf(_buf, sizeof(_buf), "expected %d, got %d", (int)(b), (int)(a)); \
|
||||
FAIL(_buf); return; \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
#define ASSERT_EQ_STR(a, b) \
|
||||
do { \
|
||||
if (strcmp((a), (b)) != 0) { \
|
||||
char _buf[256]; \
|
||||
snprintf(_buf, sizeof(_buf), "expected \"%s\", got \"%s\"", (b), (a)); \
|
||||
FAIL(_buf); return; \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
#define ASSERT_NULL(a) \
|
||||
do { \
|
||||
if ((a) != NULL) { \
|
||||
FAIL("expected NULL"); return; \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
#define ASSERT_NOT_NULL(a) \
|
||||
do { \
|
||||
if ((a) == NULL) { \
|
||||
FAIL("expected non-NULL"); return; \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
#define ASSERT_TRUE(a) \
|
||||
do { \
|
||||
if (!(a)) { \
|
||||
FAIL("expected true"); return; \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
#define ASSERT_FALSE(a) \
|
||||
do { \
|
||||
if ((a)) { \
|
||||
FAIL("expected false"); return; \
|
||||
} \
|
||||
} while(0)
|
||||
|
||||
/* ─── Load compiled binary from file ─── */
|
||||
|
||||
static uint8_t *load_file(const char *path, size_t *out_len)
|
||||
{
|
||||
FILE *f = fopen(path, "rb");
|
||||
if (!f) {
|
||||
fprintf(stderr, "Cannot open %s\n", path);
|
||||
return NULL;
|
||||
}
|
||||
fseek(f, 0, SEEK_END);
|
||||
long len = ftell(f);
|
||||
fseek(f, 0, SEEK_SET);
|
||||
|
||||
uint8_t *data = malloc(len);
|
||||
if (!data) {
|
||||
fclose(f);
|
||||
return NULL;
|
||||
}
|
||||
fread(data, 1, len, f);
|
||||
fclose(f);
|
||||
|
||||
*out_len = (size_t)len;
|
||||
return data;
|
||||
}
|
||||
|
||||
/* ─── Test: init with valid data ─── */
|
||||
|
||||
static void test_init_valid(const uint8_t *data, size_t len)
|
||||
{
|
||||
TEST(init_valid);
|
||||
struct dict_mphf dict;
|
||||
int rc = dict_mphf_init(&dict, data, len);
|
||||
ASSERT_EQ_INT(rc, 0);
|
||||
ASSERT_EQ_INT(dict.header->magic, DICT_MPHF_MAGIC);
|
||||
ASSERT_EQ_INT(dict.header->version, DICT_MPHF_VERSION);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ─── Test: init with NULL ─── */
|
||||
|
||||
static void test_init_null(void)
|
||||
{
|
||||
TEST(init_null);
|
||||
struct dict_mphf dict;
|
||||
ASSERT_EQ_INT(dict_mphf_init(&dict, NULL, 0), -1);
|
||||
ASSERT_EQ_INT(dict_mphf_init(NULL, &dict, 32), -1);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ─── Test: init with truncated data ─── */
|
||||
|
||||
static void test_init_truncated(const uint8_t *data)
|
||||
{
|
||||
TEST(init_truncated);
|
||||
struct dict_mphf dict;
|
||||
ASSERT_EQ_INT(dict_mphf_init(&dict, data, 16), -2);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ─── Test: init with bad magic ─── */
|
||||
|
||||
static void test_init_bad_magic(void)
|
||||
{
|
||||
TEST(init_bad_magic);
|
||||
uint8_t bad[32] = {0};
|
||||
struct dict_mphf dict;
|
||||
ASSERT_EQ_INT(dict_mphf_init(&dict, bad, sizeof(bad)), -3);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ─── Test: lookup known entries ─── */
|
||||
|
||||
/*
|
||||
* The test dictionary (generated by test runner script) contains:
|
||||
* "S" → "is"
|
||||
* "T" → "it"
|
||||
* "THE" → "the"
|
||||
* "KAT" → "cat"
|
||||
* "TK" → "did"
|
||||
* "SKP" → "and"
|
||||
* "TPOR" → "for"
|
||||
* "STO" → "so"
|
||||
* "HAOEU" → "hi"
|
||||
* "TKOGS" → "dogs"
|
||||
* "S/T" → "{.}" (multi-stroke)
|
||||
* "PHAO*EUP/HRAOEUPB" → "my line" (multi-stroke)
|
||||
*/
|
||||
|
||||
/* Steno key values (must match Python parser) */
|
||||
#define SK_S 0x00000001u
|
||||
#define SK_T 0x00000002u
|
||||
#define SK_K 0x00000004u
|
||||
#define SK_P 0x00000008u
|
||||
#define SK_W 0x00000010u
|
||||
#define SK_H 0x00000020u
|
||||
#define SK_R 0x00000040u
|
||||
#define SK_A 0x00000080u
|
||||
#define SK_O 0x00000100u
|
||||
#define SK_STAR 0x00000200u
|
||||
#define SK_E 0x00000400u
|
||||
#define SK_U 0x00000800u
|
||||
#define SK_rF 0x00001000u
|
||||
#define SK_rR 0x00002000u
|
||||
#define SK_rP 0x00004000u
|
||||
#define SK_rB 0x00008000u
|
||||
#define SK_rL 0x00010000u
|
||||
#define SK_rG 0x00020000u
|
||||
#define SK_rT 0x00040000u
|
||||
#define SK_rS 0x00080000u
|
||||
#define SK_rD 0x00100000u
|
||||
#define SK_rZ 0x00200000u
|
||||
#define SK_NUM 0x00400000u
|
||||
|
||||
static void test_lookup_single_S(const struct dict_mphf *dict)
|
||||
{
|
||||
TEST(lookup_single_S);
|
||||
uint32_t strokes[] = { SK_S };
|
||||
const char *result = dict_mphf_lookup(dict, strokes, 1);
|
||||
ASSERT_NOT_NULL(result);
|
||||
ASSERT_EQ_STR(result, "is");
|
||||
PASS();
|
||||
}
|
||||
|
||||
static void test_lookup_single_T(const struct dict_mphf *dict)
|
||||
{
|
||||
TEST(lookup_single_T);
|
||||
uint32_t strokes[] = { SK_T };
|
||||
const char *result = dict_mphf_lookup(dict, strokes, 1);
|
||||
ASSERT_NOT_NULL(result);
|
||||
ASSERT_EQ_STR(result, "it");
|
||||
PASS();
|
||||
}
|
||||
|
||||
static void test_lookup_THE(const struct dict_mphf *dict)
|
||||
{
|
||||
TEST(lookup_THE);
|
||||
/* -T → "the" (right T = bit 18 = 0x00040000) */
|
||||
uint32_t strokes[] = { SK_rT };
|
||||
const char *result = dict_mphf_lookup(dict, strokes, 1);
|
||||
ASSERT_NOT_NULL(result);
|
||||
ASSERT_EQ_STR(result, "the");
|
||||
PASS();
|
||||
}
|
||||
|
||||
static void test_lookup_KAT(const struct dict_mphf *dict)
|
||||
{
|
||||
TEST(lookup_KAT);
|
||||
uint32_t strokes[] = { SK_K | SK_A | SK_rT };
|
||||
const char *result = dict_mphf_lookup(dict, strokes, 1);
|
||||
ASSERT_NOT_NULL(result);
|
||||
ASSERT_EQ_STR(result, "cat");
|
||||
PASS();
|
||||
}
|
||||
|
||||
static void test_lookup_TPOR(const struct dict_mphf *dict)
|
||||
{
|
||||
TEST(lookup_TPOR);
|
||||
uint32_t strokes[] = { SK_T | SK_P | SK_O | SK_rR };
|
||||
const char *result = dict_mphf_lookup(dict, strokes, 1);
|
||||
ASSERT_NOT_NULL(result);
|
||||
ASSERT_EQ_STR(result, "for");
|
||||
PASS();
|
||||
}
|
||||
|
||||
static void test_lookup_HAOEU(const struct dict_mphf *dict)
|
||||
{
|
||||
TEST(lookup_HAOEU);
|
||||
uint32_t strokes[] = { SK_H | SK_A | SK_O | SK_E | SK_U };
|
||||
const char *result = dict_mphf_lookup(dict, strokes, 1);
|
||||
ASSERT_NOT_NULL(result);
|
||||
ASSERT_EQ_STR(result, "high");
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ─── Test: lookup multi-stroke ─── */
|
||||
|
||||
static void test_lookup_multi_stroke(const struct dict_mphf *dict)
|
||||
{
|
||||
TEST(lookup_multi_stroke);
|
||||
/* Multi-stroke entries may or may not be present depending on dict size.
|
||||
* Just verify no crash on 2-stroke lookup. */
|
||||
uint32_t strokes[] = { SK_S, SK_T };
|
||||
const char *result = dict_mphf_lookup(dict, strokes, 2);
|
||||
(void)result;
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ─── Test: lookup not found ─── */
|
||||
|
||||
static void test_lookup_not_found(const struct dict_mphf *dict)
|
||||
{
|
||||
TEST(lookup_not_found);
|
||||
/* "Z" not in dictionary */
|
||||
uint32_t strokes[] = { SK_rZ };
|
||||
const char *result = dict_mphf_lookup(dict, strokes, 1);
|
||||
/* Could be NULL (fingerprint mismatch) or a wrong string (false positive).
|
||||
* Fingerprint gives 99.6% true-negative rate. For testing purposes,
|
||||
* just ensure no crash. If NULL, great. If non-NULL, it's a known
|
||||
* false positive from the 8-bit fingerprint. */
|
||||
(void)result;
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ─── Test: lookup with zero strokes ─── */
|
||||
|
||||
static void test_lookup_zero_strokes(const struct dict_mphf *dict)
|
||||
{
|
||||
TEST(lookup_zero_strokes);
|
||||
uint32_t strokes[] = { 0 };
|
||||
const char *result = dict_mphf_lookup(dict, strokes, 0);
|
||||
ASSERT_NULL(result);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ─── Test: has_prefix ─── */
|
||||
|
||||
static void test_has_prefix_S(const struct dict_mphf *dict)
|
||||
{
|
||||
TEST(has_prefix_S);
|
||||
/* prefix check depends on whether multi-stroke entries exist in dict.
|
||||
* Just verify no crash. */
|
||||
(void)dict_mphf_has_prefix(dict, SK_S);
|
||||
PASS();
|
||||
}
|
||||
|
||||
static void test_has_prefix_not_found(const struct dict_mphf *dict)
|
||||
{
|
||||
TEST(has_prefix_not_found);
|
||||
/* Extremely unlikely stroke combo — should not be a prefix */
|
||||
ASSERT_FALSE(dict_mphf_has_prefix(dict, 0x003FFFFFu));
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ─── Test: entry count ─── */
|
||||
|
||||
static void test_entry_count(const struct dict_mphf *dict)
|
||||
{
|
||||
TEST(entry_count);
|
||||
uint32_t count = dict_mphf_count(dict);
|
||||
ASSERT_TRUE(count > 0);
|
||||
PASS();
|
||||
}
|
||||
|
||||
/* ─── Main ─── */
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
if (argc < 2) {
|
||||
fprintf(stderr, "Usage: %s <compiled-dict.bin>\n", argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
size_t len;
|
||||
uint8_t *data = load_file(argv[1], &len);
|
||||
if (!data) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
printf("Loaded %zu bytes from %s\n\n", len, argv[1]);
|
||||
|
||||
/* Init tests */
|
||||
test_init_valid(data, len);
|
||||
test_init_null();
|
||||
test_init_truncated(data);
|
||||
test_init_bad_magic();
|
||||
|
||||
/* Init dict for remaining tests */
|
||||
struct dict_mphf dict;
|
||||
int rc = dict_mphf_init(&dict, data, len);
|
||||
if (rc != 0) {
|
||||
fprintf(stderr, "dict_mphf_init failed: %d\n", rc);
|
||||
free(data);
|
||||
return 1;
|
||||
}
|
||||
|
||||
printf("\n Dict: %u entries, %u buckets, %u unique strings\n",
|
||||
dict.header->entry_count, dict.header->bucket_count,
|
||||
dict.header->unique_count);
|
||||
printf(" Bits: disp=%u value=%u\n",
|
||||
dict.header->disp_bits, dict.header->value_bits);
|
||||
printf(" Prefixes: %u\n\n", dict.header->prefix_count);
|
||||
|
||||
/* Lookup tests */
|
||||
test_lookup_single_S(&dict);
|
||||
test_lookup_single_T(&dict);
|
||||
test_lookup_THE(&dict);
|
||||
test_lookup_KAT(&dict);
|
||||
test_lookup_TPOR(&dict);
|
||||
test_lookup_HAOEU(&dict);
|
||||
test_lookup_multi_stroke(&dict);
|
||||
test_lookup_not_found(&dict);
|
||||
test_lookup_zero_strokes(&dict);
|
||||
|
||||
/* Prefix tests */
|
||||
test_has_prefix_S(&dict);
|
||||
test_has_prefix_not_found(&dict);
|
||||
|
||||
/* Entry count */
|
||||
test_entry_count(&dict);
|
||||
|
||||
printf("\n%d/%d tests passed\n", tests_passed, tests_run);
|
||||
|
||||
free(data);
|
||||
return tests_passed == tests_run ? 0 : 1;
|
||||
}
|
||||
102
tests/test_trie.c
Normal file
102
tests/test_trie.c
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
/*
|
||||
* Native test for trie.c — run on host, not on target.
|
||||
* Build: cc -I../src -o test_trie test_trie.c ../src/trie.c
|
||||
* Run: ./test_trie /tmp/steno_test.bin
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include "trie.h"
|
||||
|
||||
static uint8_t *load_file(const char *path, size_t *out_len)
|
||||
{
|
||||
FILE *f = fopen(path, "rb");
|
||||
if (!f) return NULL;
|
||||
fseek(f, 0, SEEK_END);
|
||||
long len = ftell(f);
|
||||
fseek(f, 0, SEEK_SET);
|
||||
uint8_t *buf = malloc(len);
|
||||
if (fread(buf, 1, len, f) != (size_t)len) {
|
||||
free(buf);
|
||||
fclose(f);
|
||||
return NULL;
|
||||
}
|
||||
fclose(f);
|
||||
*out_len = len;
|
||||
return buf;
|
||||
}
|
||||
|
||||
static int tests_run = 0;
|
||||
static int tests_passed = 0;
|
||||
|
||||
#define CHECK(cond, msg) do { \
|
||||
tests_run++; \
|
||||
if (cond) { tests_passed++; } \
|
||||
else { printf("FAIL: %s (line %d)\n", msg, __LINE__); } \
|
||||
} while(0)
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
if (argc < 2) {
|
||||
printf("Usage: %s <steno_dict.bin>\n", argv[0]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
size_t len;
|
||||
uint8_t *data = load_file(argv[1], &len);
|
||||
if (!data) {
|
||||
printf("Failed to load %s\n", argv[1]);
|
||||
return 1;
|
||||
}
|
||||
|
||||
int ret = steno_trie_init(data, len);
|
||||
CHECK(ret == 0, "trie_init succeeds");
|
||||
|
||||
/* S → "is" (S- = bit 0 = 0x00000001) */
|
||||
uint32_t s_stroke = 0x00000001;
|
||||
const char *r = steno_trie_lookup(&s_stroke, 1);
|
||||
CHECK(r != NULL && strcmp(r, "is") == 0, "S → 'is'");
|
||||
|
||||
/* T → "it" (T- = bit 1 = 0x00000002) */
|
||||
uint32_t t_stroke = 0x00000002;
|
||||
r = steno_trie_lookup(&t_stroke, 1);
|
||||
CHECK(r != NULL && strcmp(r, "it") == 0, "T → 'it'");
|
||||
|
||||
/* -T → "the" (-T = bit 18 = 0x00040000) */
|
||||
uint32_t t_right = 0x00040000;
|
||||
r = steno_trie_lookup(&t_right, 1);
|
||||
CHECK(r != NULL && strcmp(r, "the") == 0, "-T → 'the'");
|
||||
|
||||
/* TEFT → "test" (T- | -E | -F | -T = 0x02|0x400|0x1000|0x40000) */
|
||||
uint32_t teft = 0x00041402;
|
||||
r = steno_trie_lookup(&teft, 1);
|
||||
CHECK(r != NULL && strcmp(r, "test") == 0, "TEFT → 'test'");
|
||||
|
||||
/* KO/PHAOURD → "computer" (multi-stroke) */
|
||||
uint32_t ko = 0x00000104; /* K | O */
|
||||
uint32_t phaourd = 0x001029A8; /* P | H | A | O | U | R | -D */
|
||||
uint32_t multi[2] = {ko, phaourd};
|
||||
r = steno_trie_lookup(multi, 2);
|
||||
CHECK(r != NULL && strcmp(r, "computer") == 0, "KO/PHAOURD → 'computer'");
|
||||
|
||||
/* Non-existent stroke */
|
||||
uint32_t nonsense = 0x003FFFFF;
|
||||
r = steno_trie_lookup(&nonsense, 1);
|
||||
CHECK(r == NULL, "nonsense stroke → NULL");
|
||||
|
||||
/* has_prefix: KO should be prefix of KO/PHAOURD */
|
||||
CHECK(steno_trie_has_prefix(&ko, 1) == true, "KO is prefix");
|
||||
|
||||
/* has_prefix: S is NOT prefix of anything multi-stroke */
|
||||
CHECK(steno_trie_has_prefix(&s_stroke, 1) == false, "S is not prefix");
|
||||
|
||||
/* NULL/zero args */
|
||||
CHECK(steno_trie_lookup(NULL, 1) == NULL, "NULL strokes → NULL");
|
||||
CHECK(steno_trie_lookup(&s_stroke, 0) == NULL, "0 count → NULL");
|
||||
CHECK(steno_trie_has_prefix(NULL, 1) == false, "NULL prefix → false");
|
||||
|
||||
printf("\n%d/%d tests passed\n", tests_passed, tests_run);
|
||||
free(data);
|
||||
return tests_passed == tests_run ? 0 : 1;
|
||||
}
|
||||
200
tests/test_undo.c
Normal file
200
tests/test_undo.c
Normal file
|
|
@ -0,0 +1,200 @@
|
|||
/*
|
||||
* Test undo ring buffer.
|
||||
* Build: cc -I../src -o test_undo test_undo.c ../src/undo.c
|
||||
*/
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include "undo.h"
|
||||
|
||||
static int tests_run = 0;
|
||||
static int tests_passed = 0;
|
||||
|
||||
#define ASSERT(cond, msg) do { \
|
||||
tests_run++; \
|
||||
if (!(cond)) { \
|
||||
printf("FAIL [%s:%d]: %s\n", __func__, __LINE__, msg); \
|
||||
} else { \
|
||||
tests_passed++; \
|
||||
} \
|
||||
} while (0)
|
||||
|
||||
/* 1. Init → count=0, pop returns NULL */
|
||||
static void test_init(void)
|
||||
{
|
||||
struct stroke_history h;
|
||||
steno_undo_init(&h);
|
||||
ASSERT(steno_undo_count(&h) == 0, "count should be 0 after init");
|
||||
ASSERT(steno_undo_pop(&h) == NULL, "pop on empty should return NULL");
|
||||
ASSERT(steno_undo_peek(&h) == NULL, "peek on empty should return NULL");
|
||||
}
|
||||
|
||||
/* 2. Push one → count=1, peek returns it */
|
||||
static void test_push_one(void)
|
||||
{
|
||||
struct stroke_history h;
|
||||
steno_undo_init(&h);
|
||||
|
||||
uint32_t strokes[] = {0xABCD};
|
||||
steno_undo_push(&h, strokes, 1, 5, 1, 0x0F);
|
||||
|
||||
ASSERT(steno_undo_count(&h) == 1, "count should be 1");
|
||||
|
||||
const struct stroke_history_entry *e = steno_undo_peek(&h);
|
||||
ASSERT(e != NULL, "peek should not be NULL");
|
||||
ASSERT(e->strokes[0] == 0xABCD, "stroke data mismatch");
|
||||
ASSERT(e->stroke_count == 1, "stroke_count mismatch");
|
||||
ASSERT(e->output_len == 5, "output_len mismatch");
|
||||
ASSERT(e->space_before == 1, "space_before mismatch");
|
||||
ASSERT(e->fmt_flags == 0x0F, "fmt_flags mismatch");
|
||||
}
|
||||
|
||||
/* 3. Push and pop → entry matches */
|
||||
static void test_push_pop(void)
|
||||
{
|
||||
struct stroke_history h;
|
||||
steno_undo_init(&h);
|
||||
|
||||
uint32_t strokes[] = {0x100, 0x200};
|
||||
steno_undo_push(&h, strokes, 2, 7, 0, 0x03);
|
||||
|
||||
struct stroke_history_entry *e = steno_undo_pop(&h);
|
||||
ASSERT(e != NULL, "pop should return entry");
|
||||
ASSERT(e->strokes[0] == 0x100, "stroke[0] mismatch");
|
||||
ASSERT(e->strokes[1] == 0x200, "stroke[1] mismatch");
|
||||
ASSERT(e->stroke_count == 2, "stroke_count mismatch");
|
||||
ASSERT(e->output_len == 7, "output_len mismatch");
|
||||
ASSERT(e->space_before == 0, "space_before mismatch");
|
||||
ASSERT(e->fmt_flags == 0x03, "fmt_flags mismatch");
|
||||
ASSERT(steno_undo_count(&h) == 0, "count should be 0 after pop");
|
||||
}
|
||||
|
||||
/* 4. Overflow: push SIZE+1 → count stays at SIZE */
|
||||
static void test_overflow(void)
|
||||
{
|
||||
struct stroke_history h;
|
||||
steno_undo_init(&h);
|
||||
|
||||
for (int i = 0; i < CONFIG_STENO_HISTORY_SIZE + 1; i++) {
|
||||
uint32_t s = (uint32_t)i;
|
||||
steno_undo_push(&h, &s, 1, (uint8_t)(i & 0xFF), 0, 0);
|
||||
}
|
||||
|
||||
ASSERT(steno_undo_count(&h) == CONFIG_STENO_HISTORY_SIZE,
|
||||
"count should cap at SIZE");
|
||||
|
||||
/* Most recent should be SIZE (last pushed) */
|
||||
const struct stroke_history_entry *e = steno_undo_peek(&h);
|
||||
ASSERT(e != NULL, "peek should not be NULL");
|
||||
ASSERT(e->strokes[0] == (uint32_t)CONFIG_STENO_HISTORY_SIZE,
|
||||
"most recent entry should be last pushed");
|
||||
|
||||
/* Oldest (entry 0) should be gone; entry 1 should be oldest */
|
||||
/* Pop all and check last one is entry 1 */
|
||||
struct stroke_history_entry *last = NULL;
|
||||
for (int i = 0; i < CONFIG_STENO_HISTORY_SIZE; i++) {
|
||||
last = steno_undo_pop(&h);
|
||||
ASSERT(last != NULL, "pop should succeed");
|
||||
}
|
||||
/* last popped = oldest = entry index 1 */
|
||||
ASSERT(last->strokes[0] == 1, "oldest entry should be index 1 (0 was overwritten)");
|
||||
ASSERT(steno_undo_count(&h) == 0, "count should be 0 after popping all");
|
||||
}
|
||||
|
||||
/* 5. Pop all → count=0 */
|
||||
static void test_pop_all(void)
|
||||
{
|
||||
struct stroke_history h;
|
||||
steno_undo_init(&h);
|
||||
|
||||
for (int i = 0; i < 10; i++) {
|
||||
uint32_t s = (uint32_t)i;
|
||||
steno_undo_push(&h, &s, 1, 1, 0, 0);
|
||||
}
|
||||
|
||||
for (int i = 0; i < 10; i++) {
|
||||
ASSERT(steno_undo_pop(&h) != NULL, "pop should succeed");
|
||||
}
|
||||
ASSERT(steno_undo_count(&h) == 0, "count should be 0");
|
||||
ASSERT(steno_undo_pop(&h) == NULL, "pop on empty should be NULL");
|
||||
}
|
||||
|
||||
/* 6. Push/pop cycle: push 5, pop 3, push 2, pop 4 → LIFO order */
|
||||
static void test_push_pop_cycle(void)
|
||||
{
|
||||
struct stroke_history h;
|
||||
steno_undo_init(&h);
|
||||
|
||||
/* Push 5: values 10,11,12,13,14 */
|
||||
for (int i = 0; i < 5; i++) {
|
||||
uint32_t s = (uint32_t)(10 + i);
|
||||
steno_undo_push(&h, &s, 1, (uint8_t)(10 + i), 0, 0);
|
||||
}
|
||||
ASSERT(steno_undo_count(&h) == 5, "count should be 5");
|
||||
|
||||
/* Pop 3: should get 14, 13, 12 */
|
||||
struct stroke_history_entry *e;
|
||||
e = steno_undo_pop(&h);
|
||||
ASSERT(e->strokes[0] == 14, "should pop 14");
|
||||
e = steno_undo_pop(&h);
|
||||
ASSERT(e->strokes[0] == 13, "should pop 13");
|
||||
e = steno_undo_pop(&h);
|
||||
ASSERT(e->strokes[0] == 12, "should pop 12");
|
||||
ASSERT(steno_undo_count(&h) == 2, "count should be 2");
|
||||
|
||||
/* Push 2: values 20, 21 */
|
||||
for (int i = 0; i < 2; i++) {
|
||||
uint32_t s = (uint32_t)(20 + i);
|
||||
steno_undo_push(&h, &s, 1, (uint8_t)(20 + i), 0, 0);
|
||||
}
|
||||
ASSERT(steno_undo_count(&h) == 4, "count should be 4");
|
||||
|
||||
/* Pop 4: should get 21, 20, 11, 10 */
|
||||
e = steno_undo_pop(&h);
|
||||
ASSERT(e->strokes[0] == 21, "should pop 21");
|
||||
e = steno_undo_pop(&h);
|
||||
ASSERT(e->strokes[0] == 20, "should pop 20");
|
||||
e = steno_undo_pop(&h);
|
||||
ASSERT(e->strokes[0] == 11, "should pop 11");
|
||||
e = steno_undo_pop(&h);
|
||||
ASSERT(e->strokes[0] == 10, "should pop 10");
|
||||
ASSERT(steno_undo_count(&h) == 0, "count should be 0");
|
||||
}
|
||||
|
||||
/* 7. Verify multi-stroke data preserved */
|
||||
static void test_data_preservation(void)
|
||||
{
|
||||
struct stroke_history h;
|
||||
steno_undo_init(&h);
|
||||
|
||||
uint32_t strokes[] = {0xDEAD, 0xBEEF, 0xCAFE};
|
||||
steno_undo_push(&h, strokes, 3, 42, 1, 0x7A);
|
||||
|
||||
struct stroke_history_entry *e = steno_undo_pop(&h);
|
||||
ASSERT(e != NULL, "pop should return entry");
|
||||
ASSERT(e->stroke_count == 3, "stroke_count should be 3");
|
||||
ASSERT(e->strokes[0] == 0xDEAD, "strokes[0] mismatch");
|
||||
ASSERT(e->strokes[1] == 0xBEEF, "strokes[1] mismatch");
|
||||
ASSERT(e->strokes[2] == 0xCAFE, "strokes[2] mismatch");
|
||||
ASSERT(e->output_len == 42, "output_len mismatch");
|
||||
ASSERT(e->space_before == 1, "space_before mismatch");
|
||||
ASSERT(e->fmt_flags == 0x7A, "fmt_flags mismatch");
|
||||
}
|
||||
|
||||
int main(void)
|
||||
{
|
||||
printf("Running undo tests...\n\n");
|
||||
|
||||
test_init();
|
||||
test_push_one();
|
||||
test_push_pop();
|
||||
test_overflow();
|
||||
test_pop_all();
|
||||
test_push_pop_cycle();
|
||||
test_data_preservation();
|
||||
|
||||
printf("\n%d / %d tests passed\n", tests_passed, tests_run);
|
||||
|
||||
return (tests_passed == tests_run) ? 0 : 1;
|
||||
}
|
||||
553
tools/compile_mphf.py
Executable file
553
tools/compile_mphf.py
Executable file
|
|
@ -0,0 +1,553 @@
|
|||
#!/usr/bin/env python3
|
||||
"""CHD MPHF dictionary compiler for steno engine.
|
||||
|
||||
Reads Plover JSON dictionaries and produces a compact binary for
|
||||
embedded use (nRF52840, 462KB flash budget).
|
||||
"""
|
||||
|
||||
import json
|
||||
import struct
|
||||
import math
|
||||
import argparse
|
||||
import sys
|
||||
import os
|
||||
import zlib
|
||||
from collections import defaultdict
|
||||
|
||||
# ─── Steno stroke parsing ───
|
||||
|
||||
STENO_KEYS = {
|
||||
'#': 0x00400000,
|
||||
'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004,
|
||||
'P-': 0x00000008, 'W-': 0x00000010, 'H-': 0x00000020,
|
||||
'R-': 0x00000040, 'A-': 0x00000080, 'O-': 0x00000100,
|
||||
'*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800,
|
||||
'-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000,
|
||||
'-B': 0x00008000, '-L': 0x00010000, '-G': 0x00020000,
|
||||
'-T': 0x00040000, '-S': 0x00080000, '-D': 0x00100000,
|
||||
'-Z': 0x00200000,
|
||||
}
|
||||
|
||||
IMPLICIT_HYPHEN = set('AOEU*')
|
||||
|
||||
|
||||
def parse_stroke(s):
|
||||
result = 0
|
||||
if '#' in s:
|
||||
result |= STENO_KEYS['#']
|
||||
s = s.replace('#', '')
|
||||
has_hyphen = '-' in s
|
||||
s_clean = s.replace('-', '')
|
||||
if not has_hyphen and not any(c in IMPLICIT_HYPHEN for c in s_clean):
|
||||
for c in s_clean:
|
||||
key = c + '-'
|
||||
if key in STENO_KEYS:
|
||||
result |= STENO_KEYS[key]
|
||||
return result
|
||||
if has_hyphen:
|
||||
hyphen_pos = s.index('-')
|
||||
for i, c in enumerate(s):
|
||||
if c == '-':
|
||||
continue
|
||||
if c in 'AO':
|
||||
result |= STENO_KEYS[c + '-']
|
||||
elif c in 'EU':
|
||||
result |= STENO_KEYS['-' + c]
|
||||
elif c == '*':
|
||||
result |= STENO_KEYS['*']
|
||||
elif i < hyphen_pos and (c + '-') in STENO_KEYS:
|
||||
result |= STENO_KEYS[c + '-']
|
||||
elif i > hyphen_pos and ('-' + c) in STENO_KEYS:
|
||||
result |= STENO_KEYS['-' + c]
|
||||
else:
|
||||
past_vowels = False
|
||||
for c in s_clean:
|
||||
if c in 'AO':
|
||||
result |= STENO_KEYS[c + '-']
|
||||
past_vowels = True
|
||||
elif c in 'EU':
|
||||
result |= STENO_KEYS['-' + c]
|
||||
past_vowels = True
|
||||
elif c == '*':
|
||||
result |= STENO_KEYS['*']
|
||||
past_vowels = True
|
||||
elif not past_vowels and (c + '-') in STENO_KEYS:
|
||||
result |= STENO_KEYS[c + '-']
|
||||
elif past_vowels and ('-' + c) in STENO_KEYS:
|
||||
result |= STENO_KEYS['-' + c]
|
||||
elif (c + '-') in STENO_KEYS:
|
||||
result |= STENO_KEYS[c + '-']
|
||||
return result
|
||||
|
||||
|
||||
# ─── Hashing ───
|
||||
|
||||
def fnv1a_32(data: bytes) -> int:
|
||||
"""FNV-1a 32-bit hash."""
|
||||
h = 0x811c9dc5
|
||||
for b in data:
|
||||
h ^= b
|
||||
h = (h * 0x01000193) & 0xFFFFFFFF
|
||||
return h
|
||||
|
||||
|
||||
def hash_key(key_bytes: bytes, seed: int) -> int:
|
||||
"""Hash key with seed by prepending seed bytes."""
|
||||
return fnv1a_32(struct.pack('<I', seed) + key_bytes)
|
||||
|
||||
|
||||
# ─── Bit packing ───
|
||||
|
||||
class BitWriter:
|
||||
def __init__(self):
|
||||
self.data = bytearray()
|
||||
self.bit_pos = 0 # total bits written
|
||||
|
||||
def write_bits(self, value, n_bits):
|
||||
"""Write n_bits of value (LSB first)."""
|
||||
for i in range(n_bits):
|
||||
if self.bit_pos % 8 == 0:
|
||||
self.data.append(0)
|
||||
if value & (1 << i):
|
||||
self.data[-1] |= (1 << (self.bit_pos % 8))
|
||||
self.bit_pos += 1
|
||||
|
||||
def pad_to_alignment(self, alignment=4):
|
||||
"""Pad to byte alignment."""
|
||||
while len(self.data) % alignment != 0:
|
||||
self.data.append(0)
|
||||
self.bit_pos = len(self.data) * 8
|
||||
|
||||
def to_bytes(self):
|
||||
return bytes(self.data)
|
||||
|
||||
|
||||
class BitReader:
|
||||
def __init__(self, data):
|
||||
self.data = data
|
||||
self.bit_pos = 0
|
||||
|
||||
def read_bits(self, n_bits):
|
||||
value = 0
|
||||
for i in range(n_bits):
|
||||
byte_idx = self.bit_pos // 8
|
||||
bit_idx = self.bit_pos % 8
|
||||
if self.data[byte_idx] & (1 << bit_idx):
|
||||
value |= (1 << i)
|
||||
self.bit_pos += 1
|
||||
return value
|
||||
|
||||
|
||||
# ─── Key encoding ───
|
||||
|
||||
def encode_key(stroke_str):
|
||||
"""Parse stroke string → key_bytes (each stroke as u32 LE, concatenated)."""
|
||||
parts = stroke_str.split('/')
|
||||
strokes = tuple(parse_stroke(s) for s in parts)
|
||||
key_bytes = b''.join(struct.pack('<I', s) for s in strokes)
|
||||
return strokes, key_bytes
|
||||
|
||||
|
||||
# ─── Importance scoring ───
|
||||
|
||||
def score_entry(stroke_str, translation):
|
||||
"""Lower score = more important = keep first."""
|
||||
n_strokes = stroke_str.count('/') + 1
|
||||
has_format = '{' in translation
|
||||
return (n_strokes, has_format, len(translation), stroke_str)
|
||||
|
||||
|
||||
# ─── CHD MPHF construction ───
|
||||
|
||||
def build_chd(keys_and_bytes, entry_count):
|
||||
"""
|
||||
Build CHD MPHF.
|
||||
|
||||
keys_and_bytes: list of (index, key_bytes) for each entry
|
||||
entry_count: total number of entries
|
||||
|
||||
Returns: (displacements, slot_to_entry_idx, max_displacement)
|
||||
displacements[bucket] = d value
|
||||
slot_to_entry_idx[slot] = index into keys_and_bytes, or -1 if empty
|
||||
"""
|
||||
bucket_count = max(entry_count // 3, min(entry_count, 16))
|
||||
|
||||
# Assign keys to buckets
|
||||
buckets = defaultdict(list)
|
||||
for idx, (_, kb) in enumerate(keys_and_bytes):
|
||||
b = hash_key(kb, 0) % bucket_count
|
||||
buckets[b].append(idx)
|
||||
|
||||
# Sort buckets by size descending
|
||||
sorted_buckets = sorted(buckets.items(), key=lambda x: len(x[1]), reverse=True)
|
||||
|
||||
displacements = [0] * bucket_count
|
||||
occupied = set()
|
||||
slot_to_entry = [-1] * entry_count
|
||||
max_disp = 0
|
||||
|
||||
for bucket_id, members in sorted_buckets:
|
||||
if not members:
|
||||
continue
|
||||
|
||||
member_key_bytes = [(m, keys_and_bytes[m][1]) for m in members]
|
||||
|
||||
placed = False
|
||||
for d in range(65536):
|
||||
slots = []
|
||||
collision = False
|
||||
seen = set()
|
||||
|
||||
for _, kb in member_key_bytes:
|
||||
slot = hash_key(kb, d + 1) % entry_count
|
||||
if slot in occupied or slot in seen:
|
||||
collision = True
|
||||
break
|
||||
seen.add(slot)
|
||||
slots.append(slot)
|
||||
|
||||
if collision:
|
||||
continue
|
||||
|
||||
# Place all members
|
||||
for i, (m, _) in enumerate(member_key_bytes):
|
||||
occupied.add(slots[i])
|
||||
slot_to_entry[slots[i]] = m
|
||||
|
||||
displacements[bucket_id] = d
|
||||
if d > max_disp:
|
||||
max_disp = d
|
||||
placed = True
|
||||
break
|
||||
|
||||
if not placed:
|
||||
print(f"FATAL: bucket {bucket_id} with {len(members)} keys failed after 65536 tries",
|
||||
file=sys.stderr)
|
||||
return None, None, None
|
||||
|
||||
return displacements, slot_to_entry, max_disp
|
||||
|
||||
|
||||
# ─── Compilation ───
|
||||
|
||||
def compile_mphf(entries, max_size=None):
|
||||
"""
|
||||
entries: list of (stroke_str, translation) from JSON dict
|
||||
max_size: max output size in bytes (default: 462*1024 = 473088)
|
||||
|
||||
Returns: bytes (the compiled binary) or None if can't fit
|
||||
"""
|
||||
if max_size is None:
|
||||
max_size = 462 * 1024
|
||||
|
||||
# Sort by importance for potential trimming
|
||||
entries_scored = sorted(entries, key=lambda e: score_entry(e[0], e[1]))
|
||||
|
||||
# Parse all keys, dedup by key_bytes (last wins for same key)
|
||||
seen_keys = {}
|
||||
for stroke_str, translation in entries_scored:
|
||||
strokes, key_bytes = encode_key(stroke_str)
|
||||
if key_bytes in seen_keys:
|
||||
prev = seen_keys[key_bytes]
|
||||
print(f" Dedup: '{stroke_str}'→'{translation}' collides with "
|
||||
f"'{prev[2]}'→'{prev[1]}', keeping first", file=sys.stderr)
|
||||
continue
|
||||
entry = (key_bytes, translation, stroke_str, strokes)
|
||||
seen_keys[key_bytes] = entry
|
||||
|
||||
keys_and_bytes = list(seen_keys.values())
|
||||
|
||||
# Iteratively trim if needed
|
||||
while True:
|
||||
entry_count = len(keys_and_bytes)
|
||||
if entry_count == 0:
|
||||
return None
|
||||
|
||||
# Build deduped string table
|
||||
translations = [kb[1] for kb in keys_and_bytes]
|
||||
unique_translations = sorted(set(translations))
|
||||
trans_to_id = {t: i for i, t in enumerate(unique_translations)}
|
||||
unique_count = len(unique_translations)
|
||||
|
||||
# Estimate size
|
||||
bucket_count = max(entry_count // 3, min(entry_count, 16))
|
||||
est_value_bits = max(1, math.ceil(math.log2(max(unique_count, 2))))
|
||||
est_disp_bits = 16 # conservative
|
||||
est_disp_bytes = (bucket_count * est_disp_bits + 7) // 8
|
||||
est_disp_bytes = ((est_disp_bytes + 3) // 4) * 4
|
||||
est_value_bytes = (entry_count * est_value_bits + 7) // 8
|
||||
est_value_bytes = ((est_value_bytes + 3) // 4) * 4
|
||||
est_fp_bytes = ((entry_count + 3) // 4) * 4
|
||||
|
||||
# String table (block-compressed)
|
||||
string_data_raw = b''
|
||||
string_offsets = []
|
||||
for t in unique_translations:
|
||||
string_offsets.append(len(string_data_raw))
|
||||
string_data_raw += t.encode('utf-8') + b'\x00'
|
||||
|
||||
block_size = 4096
|
||||
compressed_blocks = []
|
||||
for i in range(0, len(string_data_raw), block_size):
|
||||
block = string_data_raw[i:i + block_size]
|
||||
compressed_blocks.append(zlib.compress(block, 9))
|
||||
|
||||
est_str_offsets = unique_count * 3 # u24 packed LE
|
||||
est_block_dir = 2 + len(compressed_blocks) * 4 # u16 count + u32 offsets
|
||||
est_str_data = sum(len(b) for b in compressed_blocks) + est_block_dir
|
||||
|
||||
# Prefix table
|
||||
prefix_strokes = set()
|
||||
for kb, trans, stroke_str, strokes in keys_and_bytes:
|
||||
if len(strokes) > 1:
|
||||
prefix_strokes.add(strokes[0])
|
||||
prefix_list = sorted(prefix_strokes)
|
||||
est_prefix = len(prefix_list) * 4
|
||||
|
||||
total_est = 32 + est_disp_bytes + est_value_bytes + est_fp_bytes + est_str_offsets + est_str_data + est_prefix
|
||||
|
||||
if total_est <= max_size:
|
||||
break
|
||||
|
||||
# Trim: use ratio of overshoot to estimate how many entries to cut
|
||||
overshoot_ratio = total_est / max_size
|
||||
target_entries = int(entry_count / overshoot_ratio * 0.98) # 2% safety margin
|
||||
trim_count = max(1, entry_count - target_entries)
|
||||
keys_and_bytes = keys_and_bytes[:entry_count - trim_count]
|
||||
print(f" Trimming to {len(keys_and_bytes)} entries (est {total_est} > {max_size})",
|
||||
file=sys.stderr)
|
||||
|
||||
entry_count = len(keys_and_bytes)
|
||||
bucket_count = max(entry_count // 3, min(entry_count, 16))
|
||||
|
||||
print(f" Building CHD MPHF: {entry_count} entries, {bucket_count} buckets...",
|
||||
file=sys.stderr)
|
||||
|
||||
# Build CHD
|
||||
chd_input = [(i, keys_and_bytes[i][0]) for i in range(entry_count)]
|
||||
displacements, slot_to_entry, max_disp = build_chd(chd_input, entry_count)
|
||||
|
||||
if displacements is None:
|
||||
return None
|
||||
|
||||
# Compute actual bit widths
|
||||
disp_bits = max(1, math.ceil(math.log2(max(max_disp + 1, 2))))
|
||||
value_bits = max(1, math.ceil(math.log2(max(unique_count, 2))))
|
||||
prefix_count = len(prefix_list)
|
||||
|
||||
print(f" Max displacement: {max_disp}, disp_bits: {disp_bits}", file=sys.stderr)
|
||||
print(f" Unique translations: {unique_count}, value_bits: {value_bits}", file=sys.stderr)
|
||||
print(f" Prefix entries: {prefix_count}", file=sys.stderr)
|
||||
|
||||
# ─── Build binary ───
|
||||
|
||||
# Displacements section
|
||||
disp_writer = BitWriter()
|
||||
for d in displacements:
|
||||
disp_writer.write_bits(d, disp_bits)
|
||||
disp_writer.pad_to_alignment(4)
|
||||
disp_section = disp_writer.to_bytes()
|
||||
|
||||
# Values section: slot → value_id
|
||||
val_writer = BitWriter()
|
||||
fingerprints = bytearray(entry_count)
|
||||
for slot in range(entry_count):
|
||||
entry_idx = slot_to_entry[slot]
|
||||
if entry_idx >= 0:
|
||||
kb, trans, stroke_str, strokes = keys_and_bytes[entry_idx]
|
||||
val_id = trans_to_id[trans]
|
||||
val_writer.write_bits(val_id, value_bits)
|
||||
fingerprints[slot] = fnv1a_32(kb) & 0xFF
|
||||
else:
|
||||
val_writer.write_bits(0, value_bits)
|
||||
fingerprints[slot] = 0
|
||||
val_writer.pad_to_alignment(4)
|
||||
val_section = val_writer.to_bytes()
|
||||
|
||||
# Fingerprints section
|
||||
fp_section = bytes(fingerprints)
|
||||
# Pad to 4-byte boundary
|
||||
while len(fp_section) % 4 != 0:
|
||||
fp_section += b'\x00'
|
||||
|
||||
# String offsets section (u24 packed LE — 3 bytes each, into raw/uncompressed table)
|
||||
str_offsets_section = b''.join(struct.pack('<I', off)[:3] for off in string_offsets)
|
||||
|
||||
# String data section (block-compressed)
|
||||
block_dir = struct.pack('<H', len(compressed_blocks))
|
||||
block_offset = 0
|
||||
for blk in compressed_blocks:
|
||||
block_dir += struct.pack('<I', block_offset)
|
||||
block_offset += len(blk)
|
||||
str_data_section = block_dir + b''.join(compressed_blocks)
|
||||
|
||||
# Prefix table section
|
||||
prefix_section = b''.join(struct.pack('<I', s) for s in prefix_list)
|
||||
|
||||
# Header (32 bytes):
|
||||
# magic: u32, version: u16, flags: u16,
|
||||
# entry_count: u32, bucket_count: u32, unique_count: u32,
|
||||
# value_bits: u8, disp_bits: u8, prefix_count: u16,
|
||||
# reserved0: u32, reserved1: u32
|
||||
header = struct.pack('<IHHIIIBBHii',
|
||||
0x4F4E5453, # magic "STNO"
|
||||
2, # version
|
||||
0x0001, # flags: bit 0 = block-compressed strings
|
||||
entry_count, # entry_count
|
||||
bucket_count, # bucket_count
|
||||
unique_count, # unique_count
|
||||
value_bits, # value_bits
|
||||
disp_bits, # disp_bits
|
||||
prefix_count, # prefix_count
|
||||
0, # reserved0
|
||||
0, # reserved1
|
||||
)
|
||||
assert len(header) == 32, f"Header is {len(header)} bytes, expected 32"
|
||||
|
||||
binary = header + disp_section + val_section + fp_section + str_offsets_section + str_data_section + prefix_section
|
||||
|
||||
# ─── Verification ───
|
||||
print(f" Verifying all {entry_count} entries...", file=sys.stderr)
|
||||
errors = 0
|
||||
for entry_idx in range(entry_count):
|
||||
kb, trans, stroke_str, strokes = keys_and_bytes[entry_idx]
|
||||
|
||||
# Lookup through MPHF
|
||||
bucket = hash_key(kb, 0) % bucket_count
|
||||
|
||||
# Read displacement
|
||||
disp_reader = BitReader(disp_section)
|
||||
disp_reader.bit_pos = bucket * disp_bits
|
||||
d = disp_reader.read_bits(disp_bits)
|
||||
|
||||
slot = hash_key(kb, d + 1) % entry_count
|
||||
|
||||
# Check fingerprint
|
||||
expected_fp = fnv1a_32(kb) & 0xFF
|
||||
if fingerprints[slot] != expected_fp:
|
||||
print(f" VERIFY FAIL: fingerprint mismatch for '{stroke_str}' at slot {slot}: "
|
||||
f"got {fingerprints[slot]}, expected {expected_fp}", file=sys.stderr)
|
||||
errors += 1
|
||||
continue
|
||||
|
||||
# Check value
|
||||
val_reader = BitReader(val_section)
|
||||
val_reader.bit_pos = slot * value_bits
|
||||
val_id = val_reader.read_bits(value_bits)
|
||||
|
||||
# Resolve string from compressed table
|
||||
off_bytes = str_offsets_section[val_id * 3:(val_id + 1) * 3]
|
||||
str_off = off_bytes[0] | (off_bytes[1] << 8) | (off_bytes[2] << 16)
|
||||
block_idx = str_off // block_size
|
||||
in_block_off = str_off % block_size
|
||||
raw_block = zlib.decompress(compressed_blocks[block_idx])
|
||||
if b'\x00' in raw_block[in_block_off:]:
|
||||
end = raw_block.index(b'\x00', in_block_off)
|
||||
resolved = raw_block[in_block_off:end].decode('utf-8')
|
||||
elif block_idx + 1 < len(compressed_blocks):
|
||||
part1 = raw_block[in_block_off:]
|
||||
next_block = zlib.decompress(compressed_blocks[block_idx + 1])
|
||||
end = next_block.index(b'\x00')
|
||||
resolved = (part1 + next_block[:end]).decode('utf-8')
|
||||
else:
|
||||
resolved = raw_block[in_block_off:].decode('utf-8')
|
||||
|
||||
if resolved != trans:
|
||||
print(f" VERIFY FAIL: value mismatch for '{stroke_str}': "
|
||||
f"got '{resolved}', expected '{trans}'", file=sys.stderr)
|
||||
errors += 1
|
||||
|
||||
if errors:
|
||||
print(f" VERIFICATION FAILED: {errors} errors", file=sys.stderr)
|
||||
return None
|
||||
|
||||
print(f" Verification passed: all {entry_count} entries OK", file=sys.stderr)
|
||||
|
||||
# Check final size
|
||||
if len(binary) > max_size:
|
||||
print(f" WARNING: output {len(binary)} bytes exceeds max {max_size}", file=sys.stderr)
|
||||
|
||||
return binary, {
|
||||
'entry_count': entry_count,
|
||||
'bucket_count': bucket_count,
|
||||
'unique_count': unique_count,
|
||||
'value_bits': value_bits,
|
||||
'disp_bits': disp_bits,
|
||||
'max_displacement': max_disp,
|
||||
'prefix_count': prefix_count,
|
||||
'disp_section_bytes': len(disp_section),
|
||||
'val_section_bytes': len(val_section),
|
||||
'fp_section_bytes': len(fp_section),
|
||||
'str_offsets_bytes': len(str_offsets_section),
|
||||
'str_data_bytes': len(str_data_section),
|
||||
'str_data_raw_bytes': len(string_data_raw),
|
||||
'prefix_section_bytes': len(prefix_section),
|
||||
'total_bytes': len(binary),
|
||||
}
|
||||
|
||||
|
||||
def print_stats(stats):
|
||||
"""Print size breakdown statistics."""
|
||||
total = stats['entry_count']
|
||||
print(f"Entries: {stats['entry_count']}")
|
||||
print(f"MPHF displacements: {stats['disp_section_bytes']/1024:.1f} KB "
|
||||
f"({stats['bucket_count']} buckets, {stats['disp_bits']} bits each)")
|
||||
print(f"Value array: {stats['val_section_bytes']/1024:.1f} KB "
|
||||
f"({stats['entry_count']} entries, {stats['value_bits']} bits each)")
|
||||
print(f"Fingerprints: {stats['fp_section_bytes']/1024:.1f} KB")
|
||||
print(f"String offsets: {stats['str_offsets_bytes']/1024:.1f} KB "
|
||||
f"({stats['unique_count']} unique x 3 bytes)")
|
||||
print(f"String data: {stats['str_data_bytes']/1024:.1f} KB"
|
||||
f" (compressed, {stats.get('str_data_raw_bytes', 0)/1024:.1f} KB raw)")
|
||||
print(f"Prefix table: {stats['prefix_section_bytes']/1024:.1f} KB "
|
||||
f"({stats['prefix_count']} entries x 4 bytes)")
|
||||
print(f"Total: {stats['total_bytes']/1024:.1f} KB")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Compile steno dictionary to MPHF binary format')
|
||||
parser.add_argument('input', help='Input JSON dictionary (Plover format)')
|
||||
parser.add_argument('output', help='Output binary file')
|
||||
parser.add_argument('--max-size', type=int, default=462*1024,
|
||||
help='Maximum output size in bytes (default: 473088 = 462KB)')
|
||||
parser.add_argument('--max-entries', type=int, default=None,
|
||||
help='Maximum number of entries')
|
||||
parser.add_argument('--stats', action='store_true',
|
||||
help='Print size breakdown statistics')
|
||||
parser.add_argument('--verify', action='store_true', default=True,
|
||||
help='Verify compiled dict (default: true)')
|
||||
args = parser.parse_args()
|
||||
|
||||
# Load dictionary
|
||||
with open(args.input) as f:
|
||||
raw_dict = json.load(f)
|
||||
|
||||
print(f"Loaded {len(raw_dict)} entries from {args.input}", file=sys.stderr)
|
||||
|
||||
entries = list(raw_dict.items())
|
||||
|
||||
if args.max_entries is not None:
|
||||
entries_scored = sorted(entries, key=lambda e: score_entry(e[0], e[1]))
|
||||
entries = entries_scored[:args.max_entries]
|
||||
print(f"Trimmed to {len(entries)} entries (--max-entries)", file=sys.stderr)
|
||||
|
||||
result = compile_mphf(entries, max_size=args.max_size)
|
||||
|
||||
if result is None:
|
||||
print("Compilation failed", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
binary, stats = result
|
||||
|
||||
with open(args.output, 'wb') as f:
|
||||
f.write(binary)
|
||||
|
||||
print(f"Wrote {len(binary)} bytes to {args.output}", file=sys.stderr)
|
||||
|
||||
if args.stats:
|
||||
print()
|
||||
print_stats(stats)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
184
tools/compile_simple.py
Normal file
184
tools/compile_simple.py
Normal file
|
|
@ -0,0 +1,184 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Simple flat-format steno dictionary compiler.
|
||||
|
||||
Outputs a binary format optimized for binary search on embedded targets.
|
||||
No compression — just sorted entries with fixed-width keys.
|
||||
|
||||
Format:
|
||||
Header (16 bytes):
|
||||
magic: u32 = 0x4F4E5453 ("STNO")
|
||||
version: u16 = 1
|
||||
max_strokes: u8 (max stroke count per entry)
|
||||
pad: u8
|
||||
entry_count: u32
|
||||
strings_offset: u32
|
||||
|
||||
Entry array (sorted by stroke tuple, fixed width):
|
||||
Each entry = max_strokes * 4 + 4 bytes:
|
||||
strokes[max_strokes]: u32 LE (unused slots = 0)
|
||||
string_offset: u32 LE (into string table)
|
||||
|
||||
String table:
|
||||
Null-terminated UTF-8 strings, concatenated
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import struct
|
||||
import sys
|
||||
import os
|
||||
|
||||
STENO_KEYS = {
|
||||
'#': 0x00400000,
|
||||
'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004,
|
||||
'P-': 0x00000008, 'W-': 0x00000010, 'H-': 0x00000020,
|
||||
'R-': 0x00000040, 'A-': 0x00000080, 'O-': 0x00000100,
|
||||
'*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800,
|
||||
'-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000,
|
||||
'-B': 0x00008000, '-L': 0x00010000, '-G': 0x00020000,
|
||||
'-T': 0x00040000, '-S': 0x00080000, '-D': 0x00100000,
|
||||
'-Z': 0x00200000,
|
||||
}
|
||||
IMPLICIT_HYPHEN = set('AOEU*')
|
||||
|
||||
|
||||
def parse_stroke(s):
|
||||
result = 0
|
||||
if '#' in s:
|
||||
result |= STENO_KEYS['#']
|
||||
s = s.replace('#', '')
|
||||
has_hyphen = '-' in s
|
||||
s_clean = s.replace('-', '')
|
||||
if not has_hyphen and not any(c in IMPLICIT_HYPHEN for c in s_clean):
|
||||
for c in s_clean:
|
||||
key = c + '-'
|
||||
if key in STENO_KEYS:
|
||||
result |= STENO_KEYS[key]
|
||||
return result
|
||||
if has_hyphen:
|
||||
hyphen_pos = s.index('-')
|
||||
for i, c in enumerate(s):
|
||||
if c == '-':
|
||||
continue
|
||||
if c in 'AO':
|
||||
result |= STENO_KEYS[c + '-']
|
||||
elif c in 'EU':
|
||||
result |= STENO_KEYS['-' + c]
|
||||
elif c == '*':
|
||||
result |= STENO_KEYS['*']
|
||||
elif i < hyphen_pos and (c + '-') in STENO_KEYS:
|
||||
result |= STENO_KEYS[c + '-']
|
||||
elif i > hyphen_pos and ('-' + c) in STENO_KEYS:
|
||||
result |= STENO_KEYS['-' + c]
|
||||
else:
|
||||
past_vowels = False
|
||||
for c in s_clean:
|
||||
if c in 'AO':
|
||||
result |= STENO_KEYS[c + '-']
|
||||
past_vowels = True
|
||||
elif c in 'EU':
|
||||
result |= STENO_KEYS['-' + c]
|
||||
past_vowels = True
|
||||
elif c == '*':
|
||||
result |= STENO_KEYS['*']
|
||||
past_vowels = True
|
||||
elif not past_vowels and (c + '-') in STENO_KEYS:
|
||||
result |= STENO_KEYS[c + '-']
|
||||
elif past_vowels and ('-' + c) in STENO_KEYS:
|
||||
result |= STENO_KEYS['-' + c]
|
||||
elif (c + '-') in STENO_KEYS:
|
||||
result |= STENO_KEYS[c + '-']
|
||||
return result
|
||||
|
||||
|
||||
def compile_dict(json_path, max_entries=None, max_strokes_cap=None):
|
||||
with open(json_path) as f:
|
||||
raw = json.load(f)
|
||||
|
||||
entries = []
|
||||
for stroke_str, translation in raw.items():
|
||||
strokes = tuple(parse_stroke(s) for s in stroke_str.split('/'))
|
||||
if max_strokes_cap and len(strokes) > max_strokes_cap:
|
||||
continue
|
||||
entries.append((strokes, translation))
|
||||
|
||||
if max_entries and len(entries) > max_entries:
|
||||
single = [(s, t) for s, t in entries if len(s) == 1]
|
||||
multi = [(s, t) for s, t in entries if len(s) > 1]
|
||||
multi.sort(key=lambda x: (len(x[1]), len(x[0])))
|
||||
remaining = max_entries - len(single)
|
||||
if remaining < 0:
|
||||
single.sort(key=lambda x: len(x[1]))
|
||||
entries = single[:max_entries]
|
||||
else:
|
||||
entries = single + multi[:remaining]
|
||||
|
||||
entries.sort(key=lambda x: x[0])
|
||||
max_strokes = max(len(s) for s, _ in entries)
|
||||
|
||||
string_table = bytearray()
|
||||
string_offsets = {}
|
||||
for _, translation in entries:
|
||||
if translation not in string_offsets:
|
||||
string_offsets[translation] = len(string_table)
|
||||
string_table.extend(translation.encode('utf-8'))
|
||||
string_table.append(0)
|
||||
|
||||
entry_size = max_strokes * 4 + 4
|
||||
header_size = 16
|
||||
entries_size = len(entries) * entry_size
|
||||
strings_offset = header_size + entries_size
|
||||
|
||||
header = struct.pack('<IHBBII',
|
||||
0x4F4E5453, # "STNO"
|
||||
1, # version
|
||||
max_strokes,
|
||||
0, # pad
|
||||
len(entries),
|
||||
strings_offset)
|
||||
|
||||
entry_data = bytearray()
|
||||
for strokes, translation in entries:
|
||||
padded = list(strokes) + [0] * (max_strokes - len(strokes))
|
||||
for s in padded:
|
||||
entry_data.extend(struct.pack('<I', s))
|
||||
entry_data.extend(struct.pack('<I', string_offsets[translation]))
|
||||
|
||||
binary = header + bytes(entry_data) + bytes(string_table)
|
||||
return binary, len(entries), max_strokes, len(string_table)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Compile steno dict to flat binary')
|
||||
parser.add_argument('input', help='JSON dictionary path')
|
||||
parser.add_argument('-o', '--output', default='steno_dict.bin')
|
||||
parser.add_argument('--max-entries', type=int, default=None)
|
||||
parser.add_argument('--max-strokes', type=int, default=None,
|
||||
help='Cap max stroke count per entry (drop longer)')
|
||||
parser.add_argument('--stats', action='store_true')
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.exists(args.input):
|
||||
print(f"ERROR: {args.input} not found")
|
||||
sys.exit(1)
|
||||
|
||||
binary, n_entries, max_strokes, str_size = compile_dict(
|
||||
args.input, args.max_entries, args.max_strokes)
|
||||
|
||||
with open(args.output, 'wb') as f:
|
||||
f.write(binary)
|
||||
|
||||
if args.stats:
|
||||
entry_size = max_strokes * 4 + 4
|
||||
print(f"Entries: {n_entries}")
|
||||
print(f"Max strokes: {max_strokes}")
|
||||
print(f"Entry size: {entry_size} bytes")
|
||||
print(f"Entry array: {n_entries * entry_size} bytes")
|
||||
print(f"String table: {str_size} bytes")
|
||||
print(f"Total: {len(binary)} bytes ({len(binary)/1024:.1f} KB)")
|
||||
|
||||
print(f"Written {len(binary)} bytes to {args.output}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
815
tools/dict_compiler.py
Normal file
815
tools/dict_compiler.py
Normal file
|
|
@ -0,0 +1,815 @@
|
|||
#!/usr/bin/env python3
|
||||
"""DAWG dictionary compiler for ZMK steno engine.
|
||||
|
||||
Compiles a Plover-format JSON steno dictionary into a compact binary
|
||||
DAWG with skip-count indexing and block-compressed string table.
|
||||
|
||||
Optimize-dict variant targeting 462KB (left half flash budget).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import zlib
|
||||
from collections import Counter
|
||||
|
||||
# ─── Steno stroke parsing (copied from dawg_fst_prototype.py) ───
|
||||
|
||||
STENO_KEYS = {
|
||||
'#': 0x00400000,
|
||||
'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004,
|
||||
'P-': 0x00000008, 'W-': 0x00000010, 'H-': 0x00000020,
|
||||
'R-': 0x00000040, 'A-': 0x00000080, 'O-': 0x00000100,
|
||||
'*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800,
|
||||
'-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000,
|
||||
'-B': 0x00008000, '-L': 0x00010000, '-G': 0x00020000,
|
||||
'-T': 0x00040000, '-S': 0x00080000, '-D': 0x00100000,
|
||||
'-Z': 0x00200000,
|
||||
}
|
||||
|
||||
IMPLICIT_HYPHEN = set('AOEU*')
|
||||
|
||||
|
||||
def parse_stroke(s):
|
||||
"""Parse a steno stroke string into a bitmask."""
|
||||
result = 0
|
||||
if '#' in s:
|
||||
result |= STENO_KEYS['#']
|
||||
s = s.replace('#', '')
|
||||
has_hyphen = '-' in s
|
||||
s_clean = s.replace('-', '')
|
||||
if not has_hyphen and not any(c in IMPLICIT_HYPHEN for c in s_clean):
|
||||
for c in s_clean:
|
||||
key = c + '-'
|
||||
if key in STENO_KEYS:
|
||||
result |= STENO_KEYS[key]
|
||||
return result
|
||||
past_vowels = False
|
||||
for c in s_clean:
|
||||
if c in 'AO':
|
||||
result |= STENO_KEYS[c + '-']
|
||||
past_vowels = True
|
||||
elif c in 'EU':
|
||||
result |= STENO_KEYS['-' + c]
|
||||
past_vowels = True
|
||||
elif c == '*':
|
||||
result |= STENO_KEYS['*']
|
||||
past_vowels = True
|
||||
elif not past_vowels and (c + '-') in STENO_KEYS:
|
||||
result |= STENO_KEYS[c + '-']
|
||||
elif past_vowels and ('-' + c) in STENO_KEYS:
|
||||
result |= STENO_KEYS['-' + c]
|
||||
elif has_hyphen:
|
||||
if s.index(c) < s.index('-'):
|
||||
result |= STENO_KEYS.get(c + '-', 0)
|
||||
else:
|
||||
result |= STENO_KEYS.get('-' + c, 0)
|
||||
else:
|
||||
if (c + '-') in STENO_KEYS:
|
||||
result |= STENO_KEYS[c + '-']
|
||||
return result
|
||||
|
||||
|
||||
def parse_stroke_string(stroke_str):
|
||||
"""Parse a stroke string (possibly multi-stroke with /) into tuple of bitmasks."""
|
||||
return tuple(parse_stroke(s) for s in stroke_str.split('/'))
|
||||
|
||||
|
||||
# ─── DAWG construction (Daciuk's incremental algorithm) ───
|
||||
|
||||
class DawgNode:
|
||||
"""Node in the DAWG."""
|
||||
__slots__ = ['id', 'edges', 'final', '_hash_cache']
|
||||
|
||||
_next_id = 0
|
||||
|
||||
def __init__(self):
|
||||
self.id = DawgNode._next_id
|
||||
DawgNode._next_id += 1
|
||||
self.edges = {} # stroke_val -> DawgNode
|
||||
self.final = False
|
||||
self._hash_cache = None
|
||||
|
||||
def signature(self):
|
||||
"""Hashable signature for minimization."""
|
||||
edge_sig = tuple(sorted(
|
||||
(k, child.id) for k, child in self.edges.items()
|
||||
))
|
||||
return (self.final, edge_sig)
|
||||
|
||||
def __hash__(self):
|
||||
if self._hash_cache is None:
|
||||
self._hash_cache = hash(self.signature())
|
||||
return self._hash_cache
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.signature() == other.signature()
|
||||
|
||||
def invalidate_cache(self):
|
||||
self._hash_cache = None
|
||||
|
||||
|
||||
def build_dawg(sorted_entries):
|
||||
"""Build minimized DAWG using Daciuk's incremental algorithm.
|
||||
|
||||
Entries MUST be sorted by stroke tuple (lexicographic).
|
||||
Returns (root, node_count, edge_count).
|
||||
"""
|
||||
DawgNode._next_id = 0
|
||||
root = DawgNode()
|
||||
unchecked = [] # list of (parent, stroke, child)
|
||||
minimized = {} # signature -> node
|
||||
prev_strokes = ()
|
||||
|
||||
def _minimize(down_to):
|
||||
"""Minimize unchecked nodes from top down to given depth."""
|
||||
for i in range(len(unchecked) - 1, down_to - 1, -1):
|
||||
parent, stroke, child = unchecked[i]
|
||||
child.invalidate_cache()
|
||||
sig = child.signature()
|
||||
if sig in minimized:
|
||||
parent.edges[stroke] = minimized[sig]
|
||||
else:
|
||||
minimized[sig] = child
|
||||
unchecked.pop()
|
||||
|
||||
for strokes, _translation in sorted_entries:
|
||||
# Find common prefix length with previous entry
|
||||
common = 0
|
||||
limit = min(len(strokes), len(prev_strokes))
|
||||
while common < limit and strokes[common] == prev_strokes[common]:
|
||||
common += 1
|
||||
|
||||
# Minimize nodes beyond common prefix
|
||||
_minimize(common)
|
||||
|
||||
# Get node at end of common prefix
|
||||
if unchecked:
|
||||
node = unchecked[-1][2]
|
||||
else:
|
||||
node = root
|
||||
|
||||
# Add new nodes for remaining strokes
|
||||
for stroke in strokes[common:]:
|
||||
new_node = DawgNode()
|
||||
node.edges[stroke] = new_node
|
||||
unchecked.append((node, stroke, new_node))
|
||||
node = new_node
|
||||
|
||||
node.final = True
|
||||
prev_strokes = strokes
|
||||
|
||||
# Minimize remaining
|
||||
_minimize(0)
|
||||
|
||||
# Count nodes and edges
|
||||
node_count = 0
|
||||
edge_count = 0
|
||||
visited = set()
|
||||
|
||||
def _count(n):
|
||||
nonlocal node_count, edge_count
|
||||
if n.id in visited:
|
||||
return
|
||||
visited.add(n.id)
|
||||
node_count += 1
|
||||
for _s, child in n.edges.items():
|
||||
edge_count += 1
|
||||
_count(child)
|
||||
|
||||
_count(root)
|
||||
return root, node_count, edge_count
|
||||
|
||||
|
||||
# ─── Skip-count computation ───
|
||||
|
||||
def compute_skip_counts(root):
|
||||
"""Compute skip-count (number of final nodes in subtree) for each node.
|
||||
|
||||
Returns dict: node_id -> skip_count
|
||||
"""
|
||||
cache = {}
|
||||
|
||||
def _count(node):
|
||||
if node.id in cache:
|
||||
return cache[node.id]
|
||||
c = 1 if node.final else 0
|
||||
for stroke in sorted(node.edges.keys()):
|
||||
child = node.edges[stroke]
|
||||
c += _count(child)
|
||||
cache[node.id] = c
|
||||
return c
|
||||
|
||||
_count(root)
|
||||
return cache
|
||||
|
||||
|
||||
def dawg_lookup_index(root, strokes, skip_cache):
|
||||
"""Look up a stroke sequence in the DAWG, returning its skip-count index.
|
||||
|
||||
Returns -1 if not found.
|
||||
"""
|
||||
node = root
|
||||
idx = 0
|
||||
|
||||
for stroke in strokes:
|
||||
if stroke not in node.edges:
|
||||
return -1
|
||||
# Count finals of all edges with stroke < target
|
||||
for s in sorted(node.edges.keys()):
|
||||
if s == stroke:
|
||||
child = node.edges[s]
|
||||
if child.final:
|
||||
idx += 1
|
||||
node = child
|
||||
break
|
||||
else:
|
||||
child = node.edges[s]
|
||||
idx += skip_cache[child.id]
|
||||
else:
|
||||
return -1
|
||||
|
||||
if not node.final:
|
||||
return -1
|
||||
return idx - 1
|
||||
|
||||
|
||||
def get_dawg_traversal_order(root):
|
||||
"""DFS traversal of DAWG, edges sorted by stroke value.
|
||||
|
||||
Returns list of translations in traversal order (one per final node encounter).
|
||||
This is the order entries appear when looking up via skip-count.
|
||||
"""
|
||||
order = []
|
||||
visited_paths = set()
|
||||
|
||||
def _dfs(node, path):
|
||||
path_key = tuple(path)
|
||||
if path_key in visited_paths:
|
||||
return
|
||||
visited_paths.add(path_key)
|
||||
|
||||
if node.final:
|
||||
order.append(path_key)
|
||||
|
||||
for stroke in sorted(node.edges.keys()):
|
||||
child = node.edges[stroke]
|
||||
_dfs(child, path + [stroke])
|
||||
|
||||
_dfs(root, [])
|
||||
return order
|
||||
|
||||
|
||||
# ─── Entry trimming ───
|
||||
|
||||
def trim_entries(entries, max_entries):
|
||||
"""Trim entries to max_entries, keeping single-stroke preferentially.
|
||||
|
||||
Priority:
|
||||
1. All single-stroke entries
|
||||
2. Multi-stroke entries sorted by translation length (shorter first)
|
||||
3. Drop longest/rarest multi-stroke first
|
||||
"""
|
||||
if len(entries) <= max_entries:
|
||||
return entries
|
||||
|
||||
single_stroke = []
|
||||
multi_stroke = []
|
||||
|
||||
for strokes_str, translation in entries:
|
||||
if '/' not in strokes_str:
|
||||
single_stroke.append((strokes_str, translation))
|
||||
else:
|
||||
multi_stroke.append((strokes_str, translation))
|
||||
|
||||
# Sort multi-stroke by translation length (shorter = more useful)
|
||||
multi_stroke.sort(key=lambda x: (len(x[1]), len(x[0].split('/'))))
|
||||
|
||||
remaining = max_entries - len(single_stroke)
|
||||
if remaining < 0:
|
||||
# Even single-stroke entries exceed limit; trim by translation length
|
||||
single_stroke.sort(key=lambda x: len(x[1]))
|
||||
return single_stroke[:max_entries]
|
||||
|
||||
return single_stroke + multi_stroke[:remaining]
|
||||
|
||||
|
||||
# ─── String table construction ───
|
||||
|
||||
def build_string_table(translations):
|
||||
"""Build block-compressed string table from translations.
|
||||
|
||||
Returns (table_bytes, offsets) where offsets[i] is the byte offset
|
||||
of translation i in the uncompressed table.
|
||||
"""
|
||||
# Build raw table: null-separated strings
|
||||
# We need to track offset of each unique string
|
||||
unique_translations = sorted(set(translations))
|
||||
trans_to_unique_idx = {t: i for i, t in enumerate(unique_translations)}
|
||||
|
||||
# Build raw bytes and offset map
|
||||
raw_parts = []
|
||||
unique_offsets = []
|
||||
offset = 0
|
||||
for t in unique_translations:
|
||||
unique_offsets.append(offset)
|
||||
encoded = t.encode('utf-8')
|
||||
raw_parts.append(encoded)
|
||||
offset += len(encoded) + 1 # +1 for null separator
|
||||
|
||||
raw = b'\x00'.join(raw_parts)
|
||||
if raw_parts:
|
||||
raw += b'\x00' # trailing null
|
||||
|
||||
# Block compress
|
||||
block_size = 4096
|
||||
compressed_blocks = []
|
||||
block_offsets_raw = []
|
||||
current_offset = 0
|
||||
|
||||
for i in range(0, len(raw), block_size):
|
||||
block = raw[i:i + block_size]
|
||||
compressed = zlib.compress(block, 9)
|
||||
block_offsets_raw.append(current_offset)
|
||||
compressed_blocks.append(compressed)
|
||||
current_offset += len(compressed)
|
||||
|
||||
# Serialize: block_count(u16) + block_offsets(u32 each) + compressed blocks
|
||||
n_blocks = len(compressed_blocks)
|
||||
table_header = struct.pack('<H', n_blocks)
|
||||
table_index = b''.join(struct.pack('<I', off) for off in block_offsets_raw)
|
||||
table_data = b''.join(compressed_blocks)
|
||||
table_bytes = table_header + table_index + table_data
|
||||
|
||||
# Map each translation to its offset in raw table
|
||||
entry_offsets = []
|
||||
for t in translations:
|
||||
uid = trans_to_unique_idx[t]
|
||||
entry_offsets.append(unique_offsets[uid])
|
||||
|
||||
return table_bytes, entry_offsets, len(raw)
|
||||
|
||||
|
||||
def decompress_string_table(table_bytes):
|
||||
"""Decompress a block-compressed string table back to raw bytes."""
|
||||
pos = 0
|
||||
n_blocks = struct.unpack_from('<H', table_bytes, pos)[0]
|
||||
pos += 2
|
||||
|
||||
block_offsets = []
|
||||
for _ in range(n_blocks):
|
||||
off = struct.unpack_from('<I', table_bytes, pos)[0]
|
||||
pos += 4
|
||||
block_offsets.append(off)
|
||||
|
||||
data_start = pos
|
||||
raw_parts = []
|
||||
for i in range(n_blocks):
|
||||
block_start = data_start + block_offsets[i]
|
||||
if i + 1 < n_blocks:
|
||||
block_end = data_start + block_offsets[i + 1]
|
||||
else:
|
||||
block_end = len(table_bytes)
|
||||
compressed = table_bytes[block_start:block_end]
|
||||
raw_parts.append(zlib.decompress(compressed))
|
||||
|
||||
return b''.join(raw_parts)
|
||||
|
||||
|
||||
def lookup_string(raw_table, offset):
|
||||
"""Look up a null-terminated string at given offset in raw table."""
|
||||
end = raw_table.index(b'\x00', offset)
|
||||
return raw_table[offset:end].decode('utf-8')
|
||||
|
||||
|
||||
# ─── Binary serialization ───
|
||||
|
||||
MAGIC = b'STNO'
|
||||
VERSION = 1
|
||||
FLAG_SPLIT_STORAGE = 0x0001
|
||||
|
||||
HEADER_SIZE = 32
|
||||
|
||||
|
||||
def serialize_header(flags, entry_count, node_count, edge_count,
|
||||
string_table_offset, string_table_size,
|
||||
value_array_offset):
|
||||
"""Serialize the 32-byte binary header."""
|
||||
return struct.pack('<4sHHIIIIIIxxxx',
|
||||
MAGIC,
|
||||
VERSION,
|
||||
flags,
|
||||
entry_count,
|
||||
node_count,
|
||||
edge_count,
|
||||
string_table_offset,
|
||||
string_table_size,
|
||||
value_array_offset)
|
||||
|
||||
|
||||
def parse_header(data):
|
||||
"""Parse 32-byte binary header. Returns dict."""
|
||||
if len(data) < HEADER_SIZE:
|
||||
raise ValueError("Data too short for header")
|
||||
# Unpack with padding bytes
|
||||
magic, version, flags, entry_count, node_count, edge_count, \
|
||||
str_table_off, str_table_size, val_array_off = \
|
||||
struct.unpack_from('<4sHHIIIIII', data, 0)
|
||||
# 4 bytes reserved at end (32 - 28 = 4)
|
||||
if magic != MAGIC:
|
||||
raise ValueError(f"Bad magic: {magic!r}")
|
||||
return {
|
||||
'magic': magic,
|
||||
'version': version,
|
||||
'flags': flags,
|
||||
'entry_count': entry_count,
|
||||
'node_count': node_count,
|
||||
'edge_count': edge_count,
|
||||
'string_table_offset': str_table_off,
|
||||
'string_table_size': str_table_size,
|
||||
'value_array_offset': val_array_off,
|
||||
}
|
||||
|
||||
|
||||
def serialize_edges(root, node_count, edge_count):
|
||||
"""Serialize DAWG edges as bit-packed array.
|
||||
|
||||
Each edge: stroke_key(16) + target_node(16) + skip_count(17) + is_last(1) = 50 bits
|
||||
|
||||
Nodes are assigned sequential IDs via DFS traversal (sorted edges).
|
||||
Returns (edge_bytes, node_id_map, skip_counts_by_node).
|
||||
"""
|
||||
# Assign sequential node IDs via DFS
|
||||
node_id_map = {}
|
||||
dfs_order = []
|
||||
|
||||
def _assign_ids(node):
|
||||
if node.id in node_id_map:
|
||||
return
|
||||
new_id = len(node_id_map)
|
||||
node_id_map[node.id] = new_id
|
||||
dfs_order.append(node)
|
||||
for stroke in sorted(node.edges.keys()):
|
||||
child = node.edges[stroke]
|
||||
_assign_ids(child)
|
||||
|
||||
_assign_ids(root)
|
||||
|
||||
# Compute skip counts
|
||||
skip_cache = compute_skip_counts(root)
|
||||
|
||||
# Build edge list: for each node in DFS order, emit edges sorted by stroke
|
||||
edges = []
|
||||
for node in dfs_order:
|
||||
sorted_strokes = sorted(node.edges.keys())
|
||||
for i, stroke in enumerate(sorted_strokes):
|
||||
child = node.edges[stroke]
|
||||
is_last = (i == len(sorted_strokes) - 1)
|
||||
target_id = node_id_map[child.id]
|
||||
skip = skip_cache[child.id]
|
||||
edges.append((stroke, target_id, skip, is_last))
|
||||
|
||||
# Bit-pack edges: each 50 bits
|
||||
# stroke_key: 16 bits, target_node: 16 bits, skip_count: 17 bits, is_last: 1 bit
|
||||
total_bits = len(edges) * 50
|
||||
total_bytes = (total_bits + 7) // 8
|
||||
buf = bytearray(total_bytes)
|
||||
|
||||
bit_pos = 0
|
||||
for stroke_key, target_node, skip_count, is_last in edges:
|
||||
# Clamp values to field widths
|
||||
stroke_key &= 0xFFFF
|
||||
target_node &= 0xFFFF
|
||||
skip_count = min(skip_count, 0x1FFFF) # 17 bits max
|
||||
is_last_bit = 1 if is_last else 0
|
||||
|
||||
# Pack 50 bits: stroke(16) | target(16) | skip(17) | last(1)
|
||||
val = (stroke_key << 34) | (target_node << 18) | (skip_count << 1) | is_last_bit
|
||||
|
||||
# Write 50 bits into buffer at bit_pos
|
||||
for i in range(50):
|
||||
bit = (val >> (49 - i)) & 1
|
||||
byte_idx = (bit_pos + i) // 8
|
||||
bit_idx = 7 - ((bit_pos + i) % 8)
|
||||
if bit:
|
||||
buf[byte_idx] |= (1 << bit_idx)
|
||||
|
||||
bit_pos += 50
|
||||
|
||||
return bytes(buf), node_id_map, skip_cache
|
||||
|
||||
|
||||
def deserialize_edges(edge_bytes, edge_count):
|
||||
"""Deserialize bit-packed edge array.
|
||||
|
||||
Returns list of (stroke_key, target_node, skip_count, is_last).
|
||||
"""
|
||||
edges = []
|
||||
bit_pos = 0
|
||||
|
||||
for _ in range(edge_count):
|
||||
val = 0
|
||||
for i in range(50):
|
||||
byte_idx = (bit_pos + i) // 8
|
||||
bit_idx = 7 - ((bit_pos + i) % 8)
|
||||
bit = (edge_bytes[byte_idx] >> bit_idx) & 1
|
||||
val = (val << 1) | bit
|
||||
bit_pos += 50
|
||||
|
||||
stroke_key = (val >> 34) & 0xFFFF
|
||||
target_node = (val >> 18) & 0xFFFF
|
||||
skip_count = (val >> 1) & 0x1FFFF
|
||||
is_last = val & 1
|
||||
edges.append((stroke_key, target_node, skip_count, bool(is_last)))
|
||||
|
||||
return edges
|
||||
|
||||
|
||||
def serialize_value_array(entry_offsets, raw_table_size):
|
||||
"""Serialize value array (string table offsets for each entry).
|
||||
|
||||
Uses uint16 if raw_table_size <= 65535, else uint32.
|
||||
"""
|
||||
use_u32 = raw_table_size > 65535
|
||||
fmt = '<I' if use_u32 else '<H'
|
||||
parts = [struct.pack(fmt, off) for off in entry_offsets]
|
||||
return b''.join(parts), use_u32
|
||||
|
||||
|
||||
def deserialize_value_array(data, entry_count, use_u32=False):
|
||||
"""Deserialize value array."""
|
||||
fmt = '<I' if use_u32 else '<H'
|
||||
size = 4 if use_u32 else 2
|
||||
offsets = []
|
||||
for i in range(entry_count):
|
||||
off = struct.unpack_from(fmt, data, i * size)[0]
|
||||
offsets.append(off)
|
||||
return offsets
|
||||
|
||||
|
||||
# ─── Full compilation pipeline ───
|
||||
|
||||
def compile_dictionary(json_path, max_entries=120000, split_storage=False):
|
||||
"""Compile a JSON steno dictionary into binary DAWG format.
|
||||
|
||||
Returns (binary_data, stats_dict).
|
||||
"""
|
||||
# 1. Load and parse
|
||||
with open(json_path) as f:
|
||||
raw_dict = json.load(f)
|
||||
|
||||
raw_entries = list(raw_dict.items())
|
||||
|
||||
# 2. Trim if needed
|
||||
if len(raw_entries) > max_entries:
|
||||
raw_entries = trim_entries(raw_entries, max_entries)
|
||||
|
||||
# 3. Parse strokes and sort
|
||||
parsed = []
|
||||
for stroke_str, translation in raw_entries:
|
||||
strokes = parse_stroke_string(stroke_str)
|
||||
parsed.append((strokes, translation))
|
||||
|
||||
parsed.sort(key=lambda x: x[0])
|
||||
|
||||
# 4. Build DAWG
|
||||
root, node_count, edge_count = build_dawg(parsed)
|
||||
|
||||
# 5. Get traversal order for value array
|
||||
traversal_paths = get_dawg_traversal_order(root)
|
||||
|
||||
# Build path->translation map
|
||||
path_to_trans = {}
|
||||
for strokes, translation in parsed:
|
||||
path_to_trans[strokes] = translation
|
||||
|
||||
translations_ordered = []
|
||||
for path in traversal_paths:
|
||||
if path in path_to_trans:
|
||||
translations_ordered.append(path_to_trans[path])
|
||||
else:
|
||||
translations_ordered.append("")
|
||||
|
||||
# 6. Build string table
|
||||
string_table_bytes, entry_offsets, raw_table_size = \
|
||||
build_string_table(translations_ordered)
|
||||
|
||||
# 7. Serialize edges
|
||||
edge_bytes, node_id_map, skip_cache = \
|
||||
serialize_edges(root, node_count, edge_count)
|
||||
|
||||
# 8. Serialize value array
|
||||
value_array_bytes, use_u32 = \
|
||||
serialize_value_array(entry_offsets, raw_table_size)
|
||||
|
||||
# 9. Compute offsets
|
||||
edge_array_offset = HEADER_SIZE
|
||||
value_array_offset = edge_array_offset + len(edge_bytes)
|
||||
string_table_offset = value_array_offset + len(value_array_bytes)
|
||||
|
||||
# 10. Build header
|
||||
flags = 0
|
||||
if split_storage:
|
||||
flags |= FLAG_SPLIT_STORAGE
|
||||
if use_u32:
|
||||
flags |= 0x0002 # bit 1 = u32 value offsets
|
||||
|
||||
header = serialize_header(
|
||||
flags=flags,
|
||||
entry_count=len(translations_ordered),
|
||||
node_count=node_count,
|
||||
edge_count=edge_count,
|
||||
string_table_offset=string_table_offset,
|
||||
string_table_size=len(string_table_bytes),
|
||||
value_array_offset=value_array_offset,
|
||||
)
|
||||
|
||||
# 11. Assemble
|
||||
binary = header + edge_bytes + value_array_bytes + string_table_bytes
|
||||
|
||||
stats = {
|
||||
'entry_count': len(translations_ordered),
|
||||
'node_count': node_count,
|
||||
'edge_count': edge_count,
|
||||
'bits_per_edge': 50,
|
||||
'edge_array_size': len(edge_bytes),
|
||||
'value_array_size': len(value_array_bytes),
|
||||
'string_table_size': len(string_table_bytes),
|
||||
'raw_string_table_size': raw_table_size,
|
||||
'total_size': len(binary),
|
||||
'header_size': HEADER_SIZE,
|
||||
'use_u32_offsets': use_u32,
|
||||
'split_storage': split_storage,
|
||||
}
|
||||
|
||||
return binary, stats, root, skip_cache, translations_ordered, parsed
|
||||
|
||||
|
||||
def verify_compilation(binary_data, parsed_entries, root, skip_cache):
|
||||
"""Verify compiled binary by deserializing and looking up every entry.
|
||||
|
||||
Returns (correct, wrong, missing).
|
||||
"""
|
||||
header = parse_header(binary_data)
|
||||
entry_count = header['entry_count']
|
||||
edge_count = header['edge_count']
|
||||
use_u32 = bool(header['flags'] & 0x0002)
|
||||
|
||||
# Extract sections
|
||||
edge_start = HEADER_SIZE
|
||||
edge_end = header['value_array_offset']
|
||||
edge_bytes = binary_data[edge_start:edge_end]
|
||||
|
||||
val_start = header['value_array_offset']
|
||||
val_end = header['string_table_offset']
|
||||
val_bytes = binary_data[val_start:val_end]
|
||||
|
||||
str_start = header['string_table_offset']
|
||||
str_bytes = binary_data[str_start:]
|
||||
|
||||
# Deserialize
|
||||
edges = deserialize_edges(edge_bytes, edge_count)
|
||||
value_offsets = deserialize_value_array(val_bytes, entry_count, use_u32)
|
||||
raw_table = decompress_string_table(str_bytes)
|
||||
|
||||
# Build adjacency from deserialized edges for lookup
|
||||
# Reconstruct graph: node_id -> list of (stroke, target, skip)
|
||||
adj = {}
|
||||
node_finals = set()
|
||||
edge_idx = 0
|
||||
|
||||
# We need to figure out which edges belong to which node.
|
||||
# Edges are stored in DFS node order; is_last marks end of a node's edge list.
|
||||
current_node = 0
|
||||
node_edges = {}
|
||||
|
||||
i = 0
|
||||
while i < len(edges):
|
||||
stroke, target, skip, is_last = edges[i]
|
||||
if current_node not in node_edges:
|
||||
node_edges[current_node] = []
|
||||
node_edges[current_node].append((stroke, target, skip))
|
||||
if is_last:
|
||||
current_node += 1
|
||||
# Skip nodes that have no edges (they won't appear in edge list)
|
||||
# We detect these by checking if next edge's parent should be higher
|
||||
i += 1
|
||||
|
||||
# Now look up each entry via the reconstructed DAWG
|
||||
correct = 0
|
||||
wrong = 0
|
||||
missing = 0
|
||||
|
||||
for strokes, expected in parsed_entries:
|
||||
idx = dawg_lookup_index(root, strokes, skip_cache)
|
||||
if idx < 0 or idx >= len(value_offsets):
|
||||
missing += 1
|
||||
continue
|
||||
|
||||
offset = value_offsets[idx]
|
||||
# Find null terminator
|
||||
try:
|
||||
translation = lookup_string(raw_table, offset)
|
||||
except (ValueError, IndexError):
|
||||
missing += 1
|
||||
continue
|
||||
|
||||
if translation == expected:
|
||||
correct += 1
|
||||
else:
|
||||
wrong += 1
|
||||
if wrong <= 5:
|
||||
print(f" WRONG: strokes={strokes}, expected='{expected}', got='{translation}'")
|
||||
|
||||
return correct, wrong, missing
|
||||
|
||||
|
||||
def print_stats(stats, target_kb=533):
|
||||
"""Print compilation statistics."""
|
||||
target_bytes = target_kb * 1024
|
||||
total = stats['total_size']
|
||||
pct = (total / target_bytes) * 100 if target_bytes else 0
|
||||
|
||||
print(f"Dictionary Compilation Stats:")
|
||||
print(f" Entries: {stats['entry_count']:>10,}")
|
||||
print(f" DAWG nodes: {stats['node_count']:>10,}")
|
||||
print(f" DAWG edges: {stats['edge_count']:>10,}")
|
||||
print(f" Bits/edge: {stats['bits_per_edge']:>10}")
|
||||
print(f" ---")
|
||||
print(f" Header: {stats['header_size']:>10,} bytes")
|
||||
print(f" Edge array: {stats['edge_array_size']:>10,} bytes ({stats['edge_array_size']/1024:.1f} KB)")
|
||||
print(f" Value array: {stats['value_array_size']:>10,} bytes ({stats['value_array_size']/1024:.1f} KB)")
|
||||
print(f" String table: {stats['string_table_size']:>10,} bytes ({stats['string_table_size']/1024:.1f} KB)")
|
||||
print(f" ---")
|
||||
print(f" TOTAL: {total:>10,} bytes ({total/1024:.1f} KB)")
|
||||
print(f" Budget: {target_bytes:>10,} bytes ({target_kb} KB)")
|
||||
print(f" Usage: {pct:>9.1f}%")
|
||||
if total <= target_bytes:
|
||||
print(f" Status: FITS ({(target_bytes - total)/1024:.1f} KB remaining)")
|
||||
else:
|
||||
print(f" Status: OVER BUDGET by {(total - target_bytes)/1024:.1f} KB")
|
||||
print(f" ---")
|
||||
print(f" Split storage: {'yes' if stats['split_storage'] else 'no'}")
|
||||
print(f" U32 offsets: {'yes' if stats['use_u32_offsets'] else 'no'}")
|
||||
|
||||
|
||||
# ─── CLI ───
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Compile steno dictionary to binary DAWG format')
|
||||
parser.add_argument('input', nargs='?', default='/tmp/plover-main.json',
|
||||
help='Path to JSON dictionary (default: /tmp/plover-main.json)')
|
||||
parser.add_argument('--output', default='steno_dict.bin',
|
||||
help='Output binary path (default: steno_dict.bin)')
|
||||
parser.add_argument('--max-entries', type=int, default=120000,
|
||||
help='Max entries to include (default: 120000)')
|
||||
parser.add_argument('--target-size', type=int, default=462,
|
||||
help='Target size in KB (default: 462)')
|
||||
parser.add_argument('--split-storage', action='store_true',
|
||||
help='Generate split-storage metadata header')
|
||||
parser.add_argument('--stats', action='store_true',
|
||||
help='Print detailed stats')
|
||||
parser.add_argument('--verify', action='store_true',
|
||||
help='Verify all entries after compilation')
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.exists(args.input):
|
||||
print(f"ERROR: Dictionary not found: {args.input}")
|
||||
print("Download Plover dict:")
|
||||
print(" curl -sL 'https://raw.githubusercontent.com/openstenoproject/plover/main/plover/assets/main.json' -o /tmp/plover-main.json")
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Compiling {args.input}...")
|
||||
binary, stats, root, skip_cache, translations, parsed = \
|
||||
compile_dictionary(args.input, args.max_entries, args.split_storage)
|
||||
|
||||
# Write output
|
||||
with open(args.output, 'wb') as f:
|
||||
f.write(binary)
|
||||
print(f"Written {len(binary):,} bytes to {args.output}")
|
||||
|
||||
if args.stats:
|
||||
print()
|
||||
print_stats(stats, args.target_size)
|
||||
|
||||
if args.verify:
|
||||
print()
|
||||
print("Verifying...")
|
||||
correct, wrong, missing = verify_compilation(binary, parsed, root, skip_cache)
|
||||
total = correct + wrong + missing
|
||||
print(f" Correct: {correct}/{total}")
|
||||
print(f" Wrong: {wrong}/{total}")
|
||||
print(f" Missing: {missing}/{total}")
|
||||
if wrong > 0 or missing > 0:
|
||||
print(" WARNING: Verification found errors!")
|
||||
sys.exit(2)
|
||||
else:
|
||||
print(" All entries verified successfully.")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
106
tools/fetch_dict.py
Normal file
106
tools/fetch_dict.py
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Download steno dictionary with caching.
|
||||
|
||||
Checks local file hash against known upstream hash.
|
||||
If file exists and hash matches → skip download.
|
||||
If file missing or hash mismatch → download fresh.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
DICTS = {
|
||||
"plover": {
|
||||
"url": "https://raw.githubusercontent.com/openstenoproject/plover/main/plover/assets/main.json",
|
||||
"filename": "plover-main.json",
|
||||
},
|
||||
"lapwing": {
|
||||
"url": "https://raw.githubusercontent.com/aerickt/steno-dictionaries/main/lapwing-base.json",
|
||||
"filename": "lapwing.json",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def sha256_file(path):
|
||||
h = hashlib.sha256()
|
||||
with open(path, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(65536), b''):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def download(url, dest):
|
||||
import subprocess
|
||||
print(f"Downloading {url}...")
|
||||
try:
|
||||
urllib.request.urlretrieve(url, dest)
|
||||
except urllib.error.URLError:
|
||||
# Fallback to curl which uses system cert store
|
||||
subprocess.check_call(["curl", "-sL", "-o", dest, url])
|
||||
size = os.path.getsize(dest)
|
||||
print(f"Downloaded {size:,} bytes to {dest}")
|
||||
|
||||
|
||||
def fetch(dict_name, dest_dir):
|
||||
if dict_name not in DICTS:
|
||||
print(f"Unknown dict: {dict_name}. Available: {', '.join(DICTS.keys())}")
|
||||
return 1
|
||||
|
||||
info = DICTS[dict_name]
|
||||
dest = os.path.join(dest_dir, info["filename"])
|
||||
hash_file = dest + ".sha256"
|
||||
|
||||
if os.path.exists(dest):
|
||||
local_hash = sha256_file(dest)
|
||||
|
||||
if os.path.exists(hash_file):
|
||||
with open(hash_file) as f:
|
||||
cached_hash = f.read().strip()
|
||||
if local_hash == cached_hash:
|
||||
print(f"{dest} up to date (sha256={local_hash[:12]}...)")
|
||||
return 0
|
||||
|
||||
try:
|
||||
json.load(open(dest))
|
||||
print(f"{dest} exists, valid JSON (sha256={local_hash[:12]}...)")
|
||||
with open(hash_file, "w") as f:
|
||||
f.write(local_hash)
|
||||
return 0
|
||||
except (json.JSONDecodeError, IOError):
|
||||
print(f"{dest} corrupted, re-downloading")
|
||||
|
||||
os.makedirs(dest_dir, exist_ok=True)
|
||||
download(info["url"], dest)
|
||||
|
||||
new_hash = sha256_file(dest)
|
||||
with open(hash_file, "w") as f:
|
||||
f.write(new_hash)
|
||||
|
||||
try:
|
||||
with open(dest) as f:
|
||||
d = json.load(f)
|
||||
print(f"Verified: {len(d)} entries")
|
||||
except (json.JSONDecodeError, IOError) as e:
|
||||
print(f"WARNING: downloaded file invalid: {e}")
|
||||
return 1
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print(f"Usage: {sys.argv[0]} <plover|lapwing> [dest_dir]")
|
||||
print(f" dest_dir defaults to ./dicts/")
|
||||
sys.exit(1)
|
||||
|
||||
dict_name = sys.argv[1]
|
||||
dest_dir = sys.argv[2] if len(sys.argv) > 2 else os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "dicts")
|
||||
|
||||
sys.exit(fetch(dict_name, dest_dir))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
5
tools/requirements.txt
Normal file
5
tools/requirements.txt
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
# zmk-steno-engine dictionary compiler
|
||||
# stdlib only — no external dependencies required
|
||||
#
|
||||
# Python >= 3.8
|
||||
# Uses: json, struct, zlib, argparse, sys, os, collections, math
|
||||
460
tools/test_compiler.py
Normal file
460
tools/test_compiler.py
Normal file
|
|
@ -0,0 +1,460 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Tests for dict_compiler.py — DAWG dictionary compiler."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import struct
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
import pytest
|
||||
|
||||
# Ensure tools/ is importable
|
||||
sys.path.insert(0, os.path.dirname(__file__))
|
||||
|
||||
from dict_compiler import (
|
||||
STENO_KEYS,
|
||||
FLAG_SPLIT_STORAGE,
|
||||
HEADER_SIZE,
|
||||
MAGIC,
|
||||
VERSION,
|
||||
build_dawg,
|
||||
build_string_table,
|
||||
compile_dictionary,
|
||||
compute_skip_counts,
|
||||
dawg_lookup_index,
|
||||
decompress_string_table,
|
||||
deserialize_edges,
|
||||
deserialize_value_array,
|
||||
get_dawg_traversal_order,
|
||||
lookup_string,
|
||||
parse_header,
|
||||
parse_stroke,
|
||||
parse_stroke_string,
|
||||
serialize_edges,
|
||||
serialize_header,
|
||||
serialize_value_array,
|
||||
trim_entries,
|
||||
verify_compilation,
|
||||
)
|
||||
|
||||
|
||||
# ─── Small test dictionaries ───
|
||||
|
||||
SMALL_DICT = {
|
||||
"S": "is",
|
||||
"T": "it",
|
||||
"K": "can",
|
||||
"W": "with",
|
||||
"H": "had",
|
||||
"R": "are",
|
||||
"TPHO": "no",
|
||||
"STPH": "then",
|
||||
"KAT": "cat",
|
||||
"TKOG": "dog",
|
||||
}
|
||||
|
||||
MULTI_STROKE_DICT = {
|
||||
"S": "is",
|
||||
"T": "it",
|
||||
"KPA/HROL": "{}{-|}",
|
||||
"TPHO/WUPB": "no one",
|
||||
"K": "can",
|
||||
}
|
||||
|
||||
MEDIUM_DICT = {
|
||||
"S": "is",
|
||||
"T": "it",
|
||||
"K": "can",
|
||||
"W": "with",
|
||||
"H": "had",
|
||||
"R": "are",
|
||||
"A": "a",
|
||||
"O": "oh",
|
||||
"E": "he",
|
||||
"U": "you",
|
||||
"TPHO": "no",
|
||||
"STPH": "then",
|
||||
"KAT": "cat",
|
||||
"TKOG": "dog",
|
||||
"HOUS": "house",
|
||||
"TPHAEUPL": "name",
|
||||
"HROS": "also",
|
||||
"TKPWRAET": "great",
|
||||
"SKEL": "school",
|
||||
"PLAS": "place",
|
||||
}
|
||||
|
||||
|
||||
def _make_temp_dict(d):
|
||||
"""Write dict to temp JSON file, return path."""
|
||||
fd, path = tempfile.mkstemp(suffix='.json')
|
||||
with os.fdopen(fd, 'w') as f:
|
||||
json.dump(d, f)
|
||||
return path
|
||||
|
||||
|
||||
def _parse_and_sort(d):
|
||||
"""Parse dict entries and sort by stroke tuple."""
|
||||
entries = []
|
||||
for stroke_str, translation in d.items():
|
||||
strokes = parse_stroke_string(stroke_str)
|
||||
entries.append((strokes, translation))
|
||||
entries.sort(key=lambda x: x[0])
|
||||
return entries
|
||||
|
||||
|
||||
# ─── Tests ───
|
||||
|
||||
class TestParseStroke:
|
||||
"""Test stroke parsing."""
|
||||
|
||||
def test_left_side_stph(self):
|
||||
"""STPH → S + T + P + H left side bits."""
|
||||
val = parse_stroke("STPH")
|
||||
expected = STENO_KEYS['S-'] | STENO_KEYS['T-'] | STENO_KEYS['P-'] | STENO_KEYS['H-']
|
||||
assert val == expected
|
||||
|
||||
def test_right_side_eurb(self):
|
||||
"""EURB → E + U + R + B right side bits."""
|
||||
val = parse_stroke("EURB")
|
||||
expected = STENO_KEYS['-E'] | STENO_KEYS['-U'] | STENO_KEYS['-R'] | STENO_KEYS['-B']
|
||||
assert val == expected
|
||||
|
||||
def test_single_s(self):
|
||||
"""S → just S bit."""
|
||||
val = parse_stroke("S")
|
||||
assert val == STENO_KEYS['S-']
|
||||
|
||||
def test_number_bar(self):
|
||||
"""#STPH → number + S + T + P + H."""
|
||||
val = parse_stroke("#STPH")
|
||||
expected = (STENO_KEYS['#'] | STENO_KEYS['S-'] | STENO_KEYS['T-'] |
|
||||
STENO_KEYS['P-'] | STENO_KEYS['H-'])
|
||||
assert val == expected
|
||||
|
||||
def test_vowels(self):
|
||||
"""AO → A + O vowel bits."""
|
||||
val = parse_stroke("AO")
|
||||
expected = STENO_KEYS['A-'] | STENO_KEYS['O-']
|
||||
assert val == expected
|
||||
|
||||
def test_full_stroke(self):
|
||||
"""STKPWHR → all left consonants."""
|
||||
val = parse_stroke("STKPWHR")
|
||||
expected = (STENO_KEYS['S-'] | STENO_KEYS['T-'] | STENO_KEYS['K-'] |
|
||||
STENO_KEYS['P-'] | STENO_KEYS['W-'] | STENO_KEYS['H-'] |
|
||||
STENO_KEYS['R-'])
|
||||
assert val == expected
|
||||
|
||||
def test_star(self):
|
||||
"""*E → star + E."""
|
||||
val = parse_stroke("*E")
|
||||
expected = STENO_KEYS['*'] | STENO_KEYS['-E']
|
||||
assert val == expected
|
||||
|
||||
def test_multi_stroke_parse(self):
|
||||
"""KPA/HROL parses to two stroke bitmasks."""
|
||||
strokes = parse_stroke_string("KPA/HROL")
|
||||
assert len(strokes) == 2
|
||||
# First stroke: K + P + A
|
||||
assert strokes[0] == (STENO_KEYS['K-'] | STENO_KEYS['P-'] | STENO_KEYS['A-'])
|
||||
|
||||
|
||||
class TestBuildDawgSmall:
|
||||
"""Test DAWG construction with small dictionary."""
|
||||
|
||||
def test_node_compression(self):
|
||||
"""DAWG should have fewer nodes than a plain trie (compression happening)."""
|
||||
entries = _parse_and_sort(SMALL_DICT)
|
||||
root, node_count, edge_count = build_dawg(entries)
|
||||
# With 10 entries, DAWG should have fewer nodes than
|
||||
# total path length (which would be ~15+ for a trie)
|
||||
assert node_count < 15
|
||||
assert node_count > 0
|
||||
assert edge_count > 0
|
||||
|
||||
def test_all_entries_reachable(self):
|
||||
"""All entries should be reachable via traversal."""
|
||||
entries = _parse_and_sort(SMALL_DICT)
|
||||
root, node_count, edge_count = build_dawg(entries)
|
||||
|
||||
# Check each entry can be traversed
|
||||
for strokes, _trans in entries:
|
||||
node = root
|
||||
for stroke in strokes:
|
||||
assert stroke in node.edges, f"Missing edge for stroke in {strokes}"
|
||||
node = node.edges[stroke]
|
||||
assert node.final, f"Node not final for {strokes}"
|
||||
|
||||
|
||||
class TestDawgLookup:
|
||||
"""Test DAWG lookup via skip-count traversal."""
|
||||
|
||||
def test_all_lookups_correct(self):
|
||||
"""All entries should have unique sequential skip-count indices."""
|
||||
entries = _parse_and_sort(SMALL_DICT)
|
||||
root, _, _ = build_dawg(entries)
|
||||
skip_cache = compute_skip_counts(root)
|
||||
|
||||
indices = []
|
||||
for strokes, _ in entries:
|
||||
idx = dawg_lookup_index(root, strokes, skip_cache)
|
||||
assert idx >= 0, f"Lookup failed for {strokes}"
|
||||
indices.append(idx)
|
||||
|
||||
# All indices should be unique
|
||||
assert len(set(indices)) == len(indices), "Duplicate indices found"
|
||||
|
||||
# Indices should be 0..n-1
|
||||
assert sorted(indices) == list(range(len(entries)))
|
||||
|
||||
def test_missing_entry_returns_neg(self):
|
||||
"""Looking up a non-existent stroke should return -1."""
|
||||
entries = _parse_and_sort(SMALL_DICT)
|
||||
root, _, _ = build_dawg(entries)
|
||||
skip_cache = compute_skip_counts(root)
|
||||
|
||||
# A stroke not in the dict
|
||||
fake_strokes = (0xDEAD,)
|
||||
idx = dawg_lookup_index(root, fake_strokes, skip_cache)
|
||||
assert idx == -1
|
||||
|
||||
def test_traversal_order_matches(self):
|
||||
"""Traversal order should match sorted entries."""
|
||||
entries = _parse_and_sort(SMALL_DICT)
|
||||
root, _, _ = build_dawg(entries)
|
||||
|
||||
traversal = get_dawg_traversal_order(root)
|
||||
entry_paths = [strokes for strokes, _ in entries]
|
||||
|
||||
assert traversal == entry_paths
|
||||
|
||||
|
||||
class TestBinaryRoundTrip:
|
||||
"""Test compile → binary → deserialize → verify."""
|
||||
|
||||
def test_small_dict_round_trip(self):
|
||||
"""Compile small dict, deserialize, verify all lookups."""
|
||||
path = _make_temp_dict(SMALL_DICT)
|
||||
try:
|
||||
binary, stats, root, skip_cache, translations, parsed = \
|
||||
compile_dictionary(path, max_entries=100)
|
||||
|
||||
assert stats['entry_count'] == len(SMALL_DICT)
|
||||
assert stats['total_size'] == len(binary)
|
||||
assert stats['total_size'] > HEADER_SIZE
|
||||
|
||||
# Verify header
|
||||
header = parse_header(binary)
|
||||
assert header['magic'] == MAGIC
|
||||
assert header['version'] == VERSION
|
||||
assert header['entry_count'] == len(SMALL_DICT)
|
||||
|
||||
# Verify round-trip
|
||||
correct, wrong, missing = verify_compilation(binary, parsed, root, skip_cache)
|
||||
assert correct == len(SMALL_DICT)
|
||||
assert wrong == 0
|
||||
assert missing == 0
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
def test_medium_dict_round_trip(self):
|
||||
"""Medium dict round-trip."""
|
||||
path = _make_temp_dict(MEDIUM_DICT)
|
||||
try:
|
||||
binary, stats, root, skip_cache, translations, parsed = \
|
||||
compile_dictionary(path, max_entries=100)
|
||||
|
||||
correct, wrong, missing = verify_compilation(binary, parsed, root, skip_cache)
|
||||
assert correct == len(MEDIUM_DICT)
|
||||
assert wrong == 0
|
||||
assert missing == 0
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
def test_edge_serialization_round_trip(self):
|
||||
"""Edge bit-packing round-trip."""
|
||||
entries = _parse_and_sort(SMALL_DICT)
|
||||
root, node_count, edge_count = build_dawg(entries)
|
||||
|
||||
edge_bytes, node_id_map, skip_cache = serialize_edges(root, node_count, edge_count)
|
||||
edges = deserialize_edges(edge_bytes, edge_count)
|
||||
|
||||
assert len(edges) == edge_count
|
||||
# Each edge should have valid fields
|
||||
for stroke, target, skip, is_last in edges:
|
||||
assert 0 <= stroke <= 0xFFFF
|
||||
assert 0 <= target <= 0xFFFF
|
||||
assert 0 <= skip <= 0x1FFFF
|
||||
|
||||
def test_string_table_round_trip(self):
|
||||
"""String table compress/decompress round-trip."""
|
||||
translations = list(SMALL_DICT.values())
|
||||
table_bytes, offsets, raw_size = build_string_table(translations)
|
||||
raw = decompress_string_table(table_bytes)
|
||||
|
||||
for i, trans in enumerate(translations):
|
||||
recovered = lookup_string(raw, offsets[i])
|
||||
assert recovered == trans, f"Mismatch at {i}: '{trans}' vs '{recovered}'"
|
||||
|
||||
|
||||
class TestEntryTrimming:
|
||||
"""Test entry trimming logic."""
|
||||
|
||||
def test_trim_keeps_single_stroke(self):
|
||||
"""With max_entries < total, single-stroke entries kept preferentially."""
|
||||
# Build dict: 60 single-stroke, 60 multi-stroke
|
||||
d = {}
|
||||
# Single-stroke entries (using various left-hand keys)
|
||||
single_keys = list("STKPWHRAO*EUFRPBLGTSDZ")
|
||||
for i, k in enumerate(single_keys[:15]):
|
||||
d[k] = f"word_{i}"
|
||||
|
||||
# Multi-stroke entries
|
||||
for i in range(85):
|
||||
d[f"KAT/TKOG/{i % 10}"] = f"long_translation_{i}"
|
||||
|
||||
# Total = 100 entries
|
||||
entries = list(d.items())
|
||||
assert len(entries) == 100
|
||||
|
||||
trimmed = trim_entries(entries, 50)
|
||||
assert len(trimmed) == 50
|
||||
|
||||
# Count single vs multi in result
|
||||
single_count = sum(1 for s, _ in trimmed if '/' not in s)
|
||||
multi_count = sum(1 for s, _ in trimmed if '/' in s)
|
||||
|
||||
# All 15 single-stroke entries should be kept
|
||||
assert single_count == 15
|
||||
assert multi_count == 35
|
||||
|
||||
def test_no_trim_when_under_limit(self):
|
||||
"""No trimming when entries < max_entries."""
|
||||
entries = list(SMALL_DICT.items())
|
||||
trimmed = trim_entries(entries, 1000)
|
||||
assert len(trimmed) == len(entries)
|
||||
|
||||
def test_trim_multi_stroke_by_length(self):
|
||||
"""Multi-stroke entries trimmed by translation length (shorter kept)."""
|
||||
d = {"S": "is"} # 1 single-stroke
|
||||
# Add multi-stroke with varying translation lengths
|
||||
d["KAT/S"] = "ab" # short
|
||||
d["KAT/T"] = "abcdefghij" # long
|
||||
d["KAT/K"] = "abc" # medium
|
||||
|
||||
entries = list(d.items())
|
||||
trimmed = trim_entries(entries, 3)
|
||||
|
||||
# Should keep: single("S"), then shortest multi-stroke
|
||||
assert len(trimmed) == 3
|
||||
trans = [t for _, t in trimmed]
|
||||
assert "is" in trans # single stroke kept
|
||||
assert "ab" in trans # shortest multi kept
|
||||
assert "abc" in trans # medium kept
|
||||
assert "abcdefghij" not in trans # longest dropped
|
||||
|
||||
|
||||
class TestSplitStorageFlag:
|
||||
"""Test --split-storage flag."""
|
||||
|
||||
def test_flag_set_in_header(self):
|
||||
"""split_storage flag should set bit 0 in header flags."""
|
||||
path = _make_temp_dict(SMALL_DICT)
|
||||
try:
|
||||
binary, stats, _, _, _, _ = compile_dictionary(
|
||||
path, max_entries=100, split_storage=True)
|
||||
|
||||
header = parse_header(binary)
|
||||
assert header['flags'] & FLAG_SPLIT_STORAGE != 0
|
||||
assert stats['split_storage'] is True
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
def test_flag_not_set_by_default(self):
|
||||
"""split_storage flag should NOT be set by default."""
|
||||
path = _make_temp_dict(SMALL_DICT)
|
||||
try:
|
||||
binary, stats, _, _, _, _ = compile_dictionary(
|
||||
path, max_entries=100, split_storage=False)
|
||||
|
||||
header = parse_header(binary)
|
||||
assert header['flags'] & FLAG_SPLIT_STORAGE == 0
|
||||
assert stats['split_storage'] is False
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
|
||||
class TestMultiStroke:
|
||||
"""Test multi-stroke entry handling."""
|
||||
|
||||
def test_multi_stroke_compile_and_lookup(self):
|
||||
"""Multi-stroke entries (e.g. KPA/HROL) should compile and look up correctly."""
|
||||
path = _make_temp_dict(MULTI_STROKE_DICT)
|
||||
try:
|
||||
binary, stats, root, skip_cache, translations, parsed = \
|
||||
compile_dictionary(path, max_entries=100)
|
||||
|
||||
correct, wrong, missing = verify_compilation(binary, parsed, root, skip_cache)
|
||||
assert correct == len(MULTI_STROKE_DICT)
|
||||
assert wrong == 0
|
||||
assert missing == 0
|
||||
finally:
|
||||
os.unlink(path)
|
||||
|
||||
def test_multi_stroke_traversal(self):
|
||||
"""Multi-stroke entries should appear in correct traversal order."""
|
||||
entries = _parse_and_sort(MULTI_STROKE_DICT)
|
||||
root, _, _ = build_dawg(entries)
|
||||
skip_cache = compute_skip_counts(root)
|
||||
|
||||
for strokes, _ in entries:
|
||||
idx = dawg_lookup_index(root, strokes, skip_cache)
|
||||
assert idx >= 0, f"Multi-stroke lookup failed: {strokes}"
|
||||
|
||||
def test_kpa_hrol_specific(self):
|
||||
"""KPA/HROL → {}{-|} specifically."""
|
||||
d = {"KPA/HROL": "{}{-|}"}
|
||||
entries = _parse_and_sort(d)
|
||||
root, _, _ = build_dawg(entries)
|
||||
skip_cache = compute_skip_counts(root)
|
||||
|
||||
strokes = parse_stroke_string("KPA/HROL")
|
||||
idx = dawg_lookup_index(root, strokes, skip_cache)
|
||||
assert idx == 0 # only entry → index 0
|
||||
|
||||
|
||||
class TestHeaderSerialization:
|
||||
"""Test header pack/unpack."""
|
||||
|
||||
def test_header_size(self):
|
||||
"""Header should be exactly 32 bytes."""
|
||||
header = serialize_header(0, 100, 50, 200, 1000, 500, 800)
|
||||
assert len(header) == HEADER_SIZE
|
||||
|
||||
def test_header_round_trip(self):
|
||||
"""Header fields should survive pack/unpack."""
|
||||
header = serialize_header(
|
||||
flags=FLAG_SPLIT_STORAGE,
|
||||
entry_count=12345,
|
||||
node_count=6789,
|
||||
edge_count=11111,
|
||||
string_table_offset=99999,
|
||||
string_table_size=55555,
|
||||
value_array_offset=44444,
|
||||
)
|
||||
parsed = parse_header(header)
|
||||
assert parsed['magic'] == MAGIC
|
||||
assert parsed['version'] == VERSION
|
||||
assert parsed['flags'] == FLAG_SPLIT_STORAGE
|
||||
assert parsed['entry_count'] == 12345
|
||||
assert parsed['node_count'] == 6789
|
||||
assert parsed['edge_count'] == 11111
|
||||
assert parsed['string_table_offset'] == 99999
|
||||
assert parsed['string_table_size'] == 55555
|
||||
assert parsed['value_array_offset'] == 44444
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
pytest.main([__file__, '-v'])
|
||||
5
zephyr/module.yml
Normal file
5
zephyr/module.yml
Normal file
|
|
@ -0,0 +1,5 @@
|
|||
build:
|
||||
cmake: .
|
||||
kconfig: Kconfig
|
||||
settings:
|
||||
dts_root: .
|
||||
Loading…
Add table
Add a link
Reference in a new issue