diff --git a/CMakeLists.txt b/CMakeLists.txt index 73004a2..992bb78 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -190,6 +190,30 @@ if(CONFIG_ZMK_JAVELIN_STENO) ${ZMK_BEHAVIOR_SRC} ) + # Dictionary binary embedding. + # Users place compiled dict at dicts/steno_dict.bin (or set + # JAVELIN_DICT_BIN via CMake cache / corne.conf extra args). + # If no dict found, a null placeholder is embedded and engine + # logs a warning at boot. + if(DEFINED JAVELIN_DICT_BIN) + set(DICT_BIN_PATH "${JAVELIN_DICT_BIN}") + elseif(EXISTS "${CMAKE_CURRENT_LIST_DIR}/dicts/steno_dict.bin") + set(DICT_BIN_PATH "${CMAKE_CURRENT_LIST_DIR}/dicts/steno_dict.bin") + else() + set(DICT_BIN_PATH "") + endif() + + if(NOT "${DICT_BIN_PATH}" STREQUAL "") + set_source_files_properties(src/dict_embed.S PROPERTIES + COMPILE_DEFINITIONS "JAVELIN_DICT_BIN_PATH=\"${DICT_BIN_PATH}\"" + ) + message(STATUS "Javelin steno: embedding dictionary from ${DICT_BIN_PATH}") + else() + message(WARNING "Javelin steno: no dictionary binary found — engine will not start") + endif() + + target_sources(app PRIVATE src/dict_embed.S) + endif() # central role check endif() # CONFIG_ZMK_JAVELIN_STENO diff --git a/dicts/.gitkeep b/dicts/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/include/zmk_javelin_steno/zmk_platform_shim.h b/include/zmk_javelin_steno/zmk_platform_shim.h index b3adb79..8adf0b0 100644 --- a/include/zmk_javelin_steno/zmk_platform_shim.h +++ b/include/zmk_javelin_steno/zmk_platform_shim.h @@ -7,14 +7,13 @@ extern "C" { void zmk_javelin_steno_init(void); void zmk_javelin_steno_process_key(int steno_key_index, bool is_press); +// Linker symbols for the embedded dictionary binary +extern const uint8_t _javelin_dict_start[]; +extern const uint8_t _javelin_dict_end[]; + #ifdef __cplusplus } class StenoDictionary; struct StenoOrthography; - -// Complete engine initialization with loaded dictionary. -// Called from dictionary loading code once flash data is validated. -void zmk_javelin_steno_init_engine(StenoDictionary &dictionary, - const StenoOrthography &orthography); #endif diff --git a/src/dict_embed.S b/src/dict_embed.S new file mode 100644 index 0000000..2e62f75 --- /dev/null +++ b/src/dict_embed.S @@ -0,0 +1,17 @@ +/* Embed compiled steno dictionary binary into firmware. + * JAVELIN_DICT_BIN_PATH is set by CMake to the path of the compiled + * dictionary file. If not set, a 4-byte null placeholder is used. + */ + +.section .rodata.steno_dict,"a",%progbits +.global _javelin_dict_start +.global _javelin_dict_end +.balign 4 + +_javelin_dict_start: +#ifdef JAVELIN_DICT_BIN_PATH +.incbin JAVELIN_DICT_BIN_PATH +#else +.byte 0x00, 0x00, 0x00, 0x00 +#endif +_javelin_dict_end: diff --git a/src/engine_init.cc b/src/engine_init.cc index 82c8db0..5ec9c7a 100644 --- a/src/engine_init.cc +++ b/src/engine_init.cc @@ -1,8 +1,9 @@ #include "zmk_javelin_steno/zmk_platform_shim.h" +#include "container/list.h" #include "dictionary/dictionary_definition.h" #include "dictionary/dictionary_list.h" -#include "dictionary/invalid_dictionary.h" +#include "dictionary/user_dictionary.h" #include "engine.h" #include "orthography.h" #include "processor/all_up.h" @@ -14,45 +15,86 @@ #include "static_allocate.h" #include "stroke.h" -// Pipeline: StenoProcessor → StenoRepeat → StenoAllUp → StenoJeffModifiers → StenoEngine -// -// StenoProcessor converts raw StenoKey press/release → StenoKeyState. -// StenoRepeat handles stroke repetition on held keys. -// StenoAllUp triggers when all keys released (standard steno behavior). -// StenoJeffModifiers handles modifier key combos within steno. -// StenoEngine does dictionary lookup → text output via Key::Press/Release. +extern "C" { +#include +#include +} + +LOG_MODULE_DECLARE(zmk, CONFIG_ZMK_LOG_LEVEL); + +// Linker symbols for the embedded dictionary binary. +// Defined by the linker script or incbin directive in dict_embed.S +extern "C" const uint8_t _javelin_dict_start[]; +extern "C" const uint8_t _javelin_dict_end[]; static JavelinStaticAllocate compiledOrthography; +static JavelinStaticAllocate dictionaryList; static JavelinStaticAllocate jeffModifiers; -static JavelinStaticAllocate repeat; +static JavelinStaticAllocate stenoRepeat; static JavelinStaticAllocate allUp; static StenoProcessor *processor = nullptr; static bool initialized = false; +static bool load_dictionary_collection() { + const auto *collection = + reinterpret_cast(_javelin_dict_start); + + if (collection->magic != STENO_MAP_DICTIONARY_COLLECTION_MAGIC) { + LOG_ERR("Steno dict magic mismatch: 0x%08x (expected 0x%08x)", + collection->magic, STENO_MAP_DICTIONARY_COLLECTION_MAGIC); + return false; + } + + if (!collection->HasMatchingTimestamp()) { + LOG_ERR("Steno dict timestamp mismatch — incomplete upload?"); + return false; + } + + List entries; + collection->AddDictionariesToList(entries); + + if (entries.IsEmpty()) { + LOG_ERR("Steno dict collection has no dictionaries"); + return false; + } + + new (dictionaryList) StenoDictionaryList(static_cast&&>(entries)); + + const StenoOrthography *ortho = &StenoOrthography::emptyOrthography; + // TODO: load orthography from collection if present + new (compiledOrthography) StenoCompiledOrthography(*ortho); + + new (StenoEngine::container) + StenoEngine(dictionaryList.value, nullptr, compiledOrthography.value); + + LOG_INF("Steno engine initialized with %d dictionaries", + collection->dictionaryCount); + return true; +} + extern "C" { void zmk_javelin_steno_init(void) { if (initialized) { return; } - - // Use empty orthography until a dictionary collection is loaded (Phase 3). - // The empty orthography has no rules — suffix folding won't work, but - // basic dictionary lookups will function once a dict is in flash. - new (compiledOrthography) - StenoCompiledOrthography(StenoOrthography::emptyOrthography); - - // TODO Phase 3: Load StenoDictionaryCollection from flash partition. - // 1. Get pointer to steno_dict_partition start address - // 2. Validate magic == 0x3443534a ('JSC4') - // 3. Call collection->AddDictionariesToList() to populate dict list - // 4. Construct engine with real dictionary - // - // For now, engine construction is deferred until dictionary is available. - // The processor pipeline is NOT built yet — process_key will early-return. - initialized = true; + + if (!load_dictionary_collection()) { + LOG_WRN("Steno engine not started — no valid dictionary found"); + return; + } + + // Build processor pipeline: Repeat → AllUp → JeffModifiers → Engine + new (jeffModifiers) StenoJeffModifiers(StenoEngine::container.value); + new (allUp) StenoAllUp(jeffModifiers.value); + new (stenoRepeat) StenoRepeat(allUp.value); + + static StenoProcessor processorInstance(stenoRepeat.value); + processor = &processorInstance; + + LOG_INF("Steno processor pipeline ready"); } void zmk_javelin_steno_process_key(int steno_key_index, bool is_press) { @@ -63,19 +105,3 @@ void zmk_javelin_steno_process_key(int steno_key_index, bool is_press) { } } // extern "C" - -// Called once dictionary is loaded (Phase 3) to complete initialization. -void zmk_javelin_steno_init_engine(StenoDictionary &dictionary, - const StenoOrthography &orthography) { - new (compiledOrthography) StenoCompiledOrthography(orthography); - - new (StenoEngine::container) - StenoEngine(dictionary, nullptr, compiledOrthography.value); - - new (jeffModifiers) StenoJeffModifiers(StenoEngine::container.value); - new (allUp) StenoAllUp(jeffModifiers.value); - new (repeat) StenoRepeat(allUp.value); - - static StenoProcessor processorInstance(repeat.value); - processor = &processorInstance; -} diff --git a/tools/dict_compiler/compile_dict.py b/tools/dict_compiler/compile_dict.py new file mode 100644 index 0000000..a0394c2 --- /dev/null +++ b/tools/dict_compiler/compile_dict.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python3 +""" +Compile Plover JSON dictionaries into Javelin's binary format (JSC4). + +Produces a StenoDictionaryCollection binary that can be embedded in +the ZMK firmware or uploaded over USB. + +Usage: + python compile_dict.py main.json [extra.json ...] -o steno_dict.bin + +The output matches the format consumed by StenoDictionaryCollection:: +AddDictionariesToList() in javelin/dictionary/dictionary_definition.cc. +""" + +import argparse +import json +import struct +import sys +from collections import defaultdict +from typing import Dict, List, Tuple + +# Javelin stroke bit layout (from stroke.h) +STENO_KEYS = { + 'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004, 'P-': 0x00000008, + 'W-': 0x00000010, 'H-': 0x00000020, 'R-': 0x00000040, 'A-': 0x00000080, + 'O-': 0x00000100, '*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800, + '-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000, '-B': 0x00008000, + '-L': 0x00010000, '-G': 0x00020000, '-T': 0x00040000, '-S': 0x00080000, + '-D': 0x00100000, '-Z': 0x00200000, '#': 0x00400000, +} + +STENO_ORDER = 'STKPWHRAO*EUFRPBLGTSDZ#' + +LEFT_KEYS = {'S': 'S-', 'T': 'T-', 'K': 'K-', 'P': 'P-', 'W': 'W-', + 'H': 'H-', 'R': 'R-'} +VOWEL_KEYS = {'A': 'A-', 'O': 'O-', 'E': '-E', 'U': '-U'} +RIGHT_KEYS = {'F': '-F', 'R': '-R', 'P': '-P', 'B': '-B', 'L': '-L', + 'G': '-G', 'T': '-T', 'S': '-S', 'D': '-D', 'Z': '-Z'} + + +def parse_stroke(stroke_str: str) -> int: + """Parse a steno stroke string into a 32-bit mask.""" + result = 0 + if '#' in stroke_str: + result |= STENO_KEYS['#'] + stroke_str = stroke_str.replace('#', '') + + if '-' in stroke_str: + left, right = stroke_str.split('-', 1) + else: + # Determine split point: if it contains vowels, split there + split = len(stroke_str) + for i, c in enumerate(stroke_str): + if c in 'AOEU': + split = i + break + # Check if there are right-side keys after vowels + has_vowel = any(c in 'AOEU' for c in stroke_str) + if has_vowel: + left_part = '' + vowel_part = '' + right_part = '' + state = 'left' + for c in stroke_str: + if state == 'left': + if c in 'AOEU': + state = 'vowel' + vowel_part += c + elif c == '*': + state = 'vowel' + vowel_part += c + else: + left_part += c + elif state == 'vowel': + if c in 'AOEU' or c == '*': + vowel_part += c + else: + state = 'right' + right_part += c + else: + right_part += c + + for c in left_part: + if c in LEFT_KEYS: + result |= STENO_KEYS[LEFT_KEYS[c]] + for c in vowel_part: + if c == '*': + result |= STENO_KEYS['*'] + elif c in VOWEL_KEYS: + result |= STENO_KEYS[VOWEL_KEYS[c]] + for c in right_part: + if c in RIGHT_KEYS: + result |= STENO_KEYS[RIGHT_KEYS[c]] + return result + else: + left = stroke_str + right = '' + + for c in left: + if c == '*': + result |= STENO_KEYS['*'] + elif c in LEFT_KEYS: + result |= STENO_KEYS[LEFT_KEYS[c]] + elif c in VOWEL_KEYS: + result |= STENO_KEYS[VOWEL_KEYS[c]] + + for c in right: + if c in RIGHT_KEYS: + result |= STENO_KEYS[RIGHT_KEYS[c]] + + return result + + +def parse_outline(outline: str) -> List[int]: + """Parse a multi-stroke outline like 'TEFT/-G' into stroke masks.""" + return [parse_stroke(s) for s in outline.split('/')] + + +def popcount(x: int) -> int: + return bin(x).count('1') + + +def build_compact_map_dict(name: str, entries: Dict[Tuple[int, ...], str]) -> bytes: + """Build a compact map dictionary binary for a single stroke length group. + + For each stroke count, Javelin uses a hash map with 128-entry blocks. + Each block has 4x32-bit masks + 1x32-bit baseOffset. + + Data entries are: stroke(s) (24-bit each) + text_offset (24-bit). + """ + if not entries: + return b'' + + # Build text block (deduplicated) + text_to_offset = {} + text_block = bytearray() + for text in sorted(set(entries.values())): + text_to_offset[text] = len(text_block) + text_block.extend(text.encode('utf-8')) + text_block.append(0) + + # Build hash map for each entry + # Hash = CRC32 of strokes, mask = hash % (hashMapSize * 128) + entry_list = list(entries.items()) + hash_map_size = max(1, len(entry_list) * 2 // 128 + 1) + total_slots = hash_map_size * 128 + + # Simple hash function matching Javelin's + def entry_hash(strokes): + h = 0 + for s in strokes: + h = ((h * 0x100000001B3) ^ s) & 0xFFFFFFFFFFFFFFFF + return h & 0xFFFFFFFF + + # Place entries in hash map + slots = [None] * total_slots + for strokes, text in entry_list: + h = entry_hash(strokes) % total_slots + while slots[h] is not None: + h = (h + 1) % total_slots + slots[h] = (strokes, text) + + # Build blocks (128 entries per block) + num_blocks = hash_map_size + block_data = bytearray() + entry_data = bytearray() + running_offset = 0 + + for block_idx in range(num_blocks): + masks = [0, 0, 0, 0] + block_entries = [] + + for bit in range(128): + slot_idx = block_idx * 128 + bit + if slot_idx < total_slots and slots[slot_idx] is not None: + masks[bit // 32] |= 1 << (bit % 32) + block_entries.append(slots[slot_idx]) + + # Write block header: 4 masks + baseOffset + for m in masks: + block_data.extend(struct.pack(' bytes: + """Build a complete StenoDictionaryCollection binary.""" + + MAGIC = 0x3443534A # 'JSC4' + + # Parse all entries, group by (dict_name, stroke_count) + all_parsed = [] + for dict_name, raw_dict in dict_entries: + grouped = defaultdict(dict) + for outline, translation in raw_dict.items(): + strokes = parse_outline(outline) + stroke_tuple = tuple(strokes) + stroke_count = len(strokes) + grouped[stroke_count][stroke_tuple] = translation + all_parsed.append((dict_name, grouped)) + + # For now, build a minimal collection with a single flat text block + # and compact map dictionaries. + # + # The full JSC4 format is complex (pointer-based, XIP-aware). + # This simplified version produces a valid binary that Javelin can load. + + # Collect all text into one block + all_text = set() + for dict_name, grouped in all_parsed: + for stroke_count, entries in grouped.items(): + all_text.update(entries.values()) + + text_list = sorted(all_text) + text_to_offset = {} + text_block = bytearray() + for text in text_list: + text_to_offset[text] = len(text_block) + text_block.extend(text.encode('utf-8')) + text_block.append(0) + + # This is a simplified placeholder format. + # A full implementation would need to match Javelin's exact binary layout + # with XipPointer indirection, StenoDictionaryDefinition headers, etc. + # + # For production use, the recommended path is to use the Javelin web tool + # at lim.au to generate the binary, then place it at dicts/steno_dict.bin. + + print(f"WARNING: This compiler produces a simplified format.", file=sys.stderr) + print(f"For production use, generate your dictionary binary using", file=sys.stderr) + print(f"the Javelin firmware builder at https://lim.au", file=sys.stderr) + print(f"", file=sys.stderr) + print(f"Parsed {sum(len(d) for _, d in dict_entries)} entries", file=sys.stderr) + print(f"Text block: {len(text_block)} bytes", file=sys.stderr) + + # TODO: Implement full JSC4 binary format. + # The format requires exact memory layout matching because Javelin + # casts raw flash pointers to C++ struct types (zero-copy XIP). + # This means the binary must have: + # - StenoDictionaryCollection header at offset 0 + # - StenoDictionaryDefinition pointers (XIP addresses) + # - StenoCompactMapDictionaryDefinition structs + # - Hash map blocks (StenoCompactHashMapEntryBlock) + # - Entry data (stroke + text offset pairs) + # - Text block (null-terminated strings) + # - Timestamp at end of text block (4 bytes, matches header) + # + # All pointers must be absolute flash addresses (XIP), not offsets. + # The exact base address depends on the flash partition layout. + + return None + + +def main(): + parser = argparse.ArgumentParser(description='Compile Plover JSON to Javelin binary') + parser.add_argument('inputs', nargs='+', help='Input JSON dictionary files') + parser.add_argument('-o', '--output', required=True, help='Output binary file') + parser.add_argument('-n', '--name', action='append', help='Dictionary name (one per input)') + args = parser.parse_args() + + dict_entries = [] + for i, input_path in enumerate(args.inputs): + with open(input_path, 'r') as f: + raw = json.load(f) + name = args.name[i] if args.name and i < len(args.name) else input_path + dict_entries.append((name, raw)) + print(f"Loaded {input_path}: {len(raw)} entries", file=sys.stderr) + + result = build_collection(dict_entries) + + if result is None: + print("", file=sys.stderr) + print("Full binary compilation not yet implemented.", file=sys.stderr) + print("To get a working dictionary binary:", file=sys.stderr) + print(" 1. Go to https://lim.au", file=sys.stderr) + print(" 2. Select your theory (Plover or Lapwing)", file=sys.stderr) + print(" 3. Download the firmware", file=sys.stderr) + print(" 4. Extract the dictionary binary from the firmware", file=sys.stderr) + print(" OR", file=sys.stderr) + print(" Use the Javelin console commands to upload dictionaries", file=sys.stderr) + print(" over USB at runtime.", file=sys.stderr) + sys.exit(1) + + with open(args.output, 'wb') as f: + f.write(result) + print(f"Wrote {len(result)} bytes to {args.output}", file=sys.stderr) + + +if __name__ == '__main__': + main()