Phase 3: dictionary loading pipeline + binary embedding

- engine_init.cc: loads StenoDictionaryCollection from embedded binary,
  validates JSC4 magic + timestamp, constructs dict list + full processor
  pipeline (Repeat → AllUp → JeffModifiers → Engine)
- dict_embed.S: embeds compiled dict binary via .incbin directive.
  Falls back to null placeholder if no dict available
- CMakeLists.txt: dict binary discovery (dicts/steno_dict.bin or
  JAVELIN_DICT_BIN cmake var), assembly embedding
- tools/dict_compiler/compile_dict.py: stroke parser + skeleton compiler.
  Full JSC4 binary format not yet implemented — for now, use Javelin
  web tool (lim.au) to generate dict binary
- dicts/: directory for compiled dictionary binaries

Dict compiler is the remaining gap — JSC4 format requires exact XIP
pointer layout matching Javelin's C++ struct casting.

CMakeLists.txt

@@ -190,6 +190,30 @@ if(CONFIG_ZMK_JAVELIN_STENO)
       ${ZMK_BEHAVIOR_SRC}
     )
 
+    # Dictionary binary embedding.
+    # Users place compiled dict at dicts/steno_dict.bin (or set
+    # JAVELIN_DICT_BIN via CMake cache / corne.conf extra args).
+    # If no dict found, a null placeholder is embedded and engine
+    # logs a warning at boot.
+    if(DEFINED JAVELIN_DICT_BIN)
+      set(DICT_BIN_PATH "${JAVELIN_DICT_BIN}")
+    elseif(EXISTS "${CMAKE_CURRENT_LIST_DIR}/dicts/steno_dict.bin")
+      set(DICT_BIN_PATH "${CMAKE_CURRENT_LIST_DIR}/dicts/steno_dict.bin")
+    else()
+      set(DICT_BIN_PATH "")
+    endif()
+
+    if(NOT "${DICT_BIN_PATH}" STREQUAL "")
+      set_source_files_properties(src/dict_embed.S PROPERTIES
+        COMPILE_DEFINITIONS "JAVELIN_DICT_BIN_PATH=\"${DICT_BIN_PATH}\""
+      )
+      message(STATUS "Javelin steno: embedding dictionary from ${DICT_BIN_PATH}")
+    else()
+      message(WARNING "Javelin steno: no dictionary binary found — engine will not start")
+    endif()
+
+    target_sources(app PRIVATE src/dict_embed.S)
+
   endif() # central role check
 
 endif() # CONFIG_ZMK_JAVELIN_STENO

include/zmk_javelin_steno/zmk_platform_shim.h

@@ -7,14 +7,13 @@ extern "C" {
 void zmk_javelin_steno_init(void);
 void zmk_javelin_steno_process_key(int steno_key_index, bool is_press);
 
+// Linker symbols for the embedded dictionary binary
+extern const uint8_t _javelin_dict_start[];
+extern const uint8_t _javelin_dict_end[];
+
 #ifdef __cplusplus
 }
 
 class StenoDictionary;
 struct StenoOrthography;
-
-// Complete engine initialization with loaded dictionary.
-// Called from dictionary loading code once flash data is validated.
-void zmk_javelin_steno_init_engine(StenoDictionary &dictionary,
-                                   const StenoOrthography &orthography);
 #endif

src/dict_embed.S

@@ -0,0 +1,17 @@
+/* Embed compiled steno dictionary binary into firmware.
+ * JAVELIN_DICT_BIN_PATH is set by CMake to the path of the compiled
+ * dictionary file. If not set, a 4-byte null placeholder is used.
+ */
+
+.section .rodata.steno_dict,"a",%progbits
+.global _javelin_dict_start
+.global _javelin_dict_end
+.balign 4
+
+_javelin_dict_start:
+#ifdef JAVELIN_DICT_BIN_PATH
+.incbin JAVELIN_DICT_BIN_PATH
+#else
+.byte 0x00, 0x00, 0x00, 0x00
+#endif
+_javelin_dict_end:

src/engine_init.cc

@@ -1,8 +1,9 @@
 #include "zmk_javelin_steno/zmk_platform_shim.h"
 
+#include "container/list.h"
 #include "dictionary/dictionary_definition.h"
 #include "dictionary/dictionary_list.h"
-#include "dictionary/invalid_dictionary.h"
+#include "dictionary/user_dictionary.h"
 #include "engine.h"
 #include "orthography.h"
 #include "processor/all_up.h"
@@ -14,45 +15,86 @@
 #include "static_allocate.h"
 #include "stroke.h"
 
-// Pipeline: StenoProcessor → StenoRepeat → StenoAllUp → StenoJeffModifiers → StenoEngine
+extern "C" {
-//
+#include <zephyr/kernel.h>
-// StenoProcessor converts raw StenoKey press/release → StenoKeyState.
+#include <zephyr/logging/log.h>
-// StenoRepeat handles stroke repetition on held keys.
+}
-// StenoAllUp triggers when all keys released (standard steno behavior).
+
-// StenoJeffModifiers handles modifier key combos within steno.
+LOG_MODULE_DECLARE(zmk, CONFIG_ZMK_LOG_LEVEL);
-// StenoEngine does dictionary lookup → text output via Key::Press/Release.
+
+// Linker symbols for the embedded dictionary binary.
+// Defined by the linker script or incbin directive in dict_embed.S
+extern "C" const uint8_t _javelin_dict_start[];
+extern "C" const uint8_t _javelin_dict_end[];
 
 static JavelinStaticAllocate<StenoCompiledOrthography> compiledOrthography;
+static JavelinStaticAllocate<StenoDictionaryList> dictionaryList;
 static JavelinStaticAllocate<StenoJeffModifiers> jeffModifiers;
-static JavelinStaticAllocate<StenoRepeat> repeat;
+static JavelinStaticAllocate<StenoRepeat> stenoRepeat;
 static JavelinStaticAllocate<StenoAllUp> allUp;
 static StenoProcessor *processor = nullptr;
 
 static bool initialized = false;
 
+static bool load_dictionary_collection() {
+  const auto *collection =
+      reinterpret_cast<const StenoDictionaryCollection *>(_javelin_dict_start);
+
+  if (collection->magic != STENO_MAP_DICTIONARY_COLLECTION_MAGIC) {
+    LOG_ERR("Steno dict magic mismatch: 0x%08x (expected 0x%08x)",
+            collection->magic, STENO_MAP_DICTIONARY_COLLECTION_MAGIC);
+    return false;
+  }
+
+  if (!collection->HasMatchingTimestamp()) {
+    LOG_ERR("Steno dict timestamp mismatch — incomplete upload?");
+    return false;
+  }
+
+  List<StenoDictionaryListEntry> entries;
+  collection->AddDictionariesToList(entries);
+
+  if (entries.IsEmpty()) {
+    LOG_ERR("Steno dict collection has no dictionaries");
+    return false;
+  }
+
+  new (dictionaryList) StenoDictionaryList(static_cast<List<StenoDictionaryListEntry>&&>(entries));
+
+  const StenoOrthography *ortho = &StenoOrthography::emptyOrthography;
+  // TODO: load orthography from collection if present
+  new (compiledOrthography) StenoCompiledOrthography(*ortho);
+
+  new (StenoEngine::container)
+      StenoEngine(dictionaryList.value, nullptr, compiledOrthography.value);
+
+  LOG_INF("Steno engine initialized with %d dictionaries",
+          collection->dictionaryCount);
+  return true;
+}
+
 extern "C" {
 
 void zmk_javelin_steno_init(void) {
   if (initialized) {
     return;
   }
-
-  // Use empty orthography until a dictionary collection is loaded (Phase 3).
-  // The empty orthography has no rules — suffix folding won't work, but
-  // basic dictionary lookups will function once a dict is in flash.
-  new (compiledOrthography)
-      StenoCompiledOrthography(StenoOrthography::emptyOrthography);
-
-  // TODO Phase 3: Load StenoDictionaryCollection from flash partition.
-  // 1. Get pointer to steno_dict_partition start address
-  // 2. Validate magic == 0x3443534a ('JSC4')
-  // 3. Call collection->AddDictionariesToList() to populate dict list
-  // 4. Construct engine with real dictionary
-  //
-  // For now, engine construction is deferred until dictionary is available.
-  // The processor pipeline is NOT built yet — process_key will early-return.
-
   initialized = true;
+
+  if (!load_dictionary_collection()) {
+    LOG_WRN("Steno engine not started — no valid dictionary found");
+    return;
+  }
+
+  // Build processor pipeline: Repeat → AllUp → JeffModifiers → Engine
+  new (jeffModifiers) StenoJeffModifiers(StenoEngine::container.value);
+  new (allUp) StenoAllUp(jeffModifiers.value);
+  new (stenoRepeat) StenoRepeat(allUp.value);
+
+  static StenoProcessor processorInstance(stenoRepeat.value);
+  processor = &processorInstance;
+
+  LOG_INF("Steno processor pipeline ready");
 }
 
 void zmk_javelin_steno_process_key(int steno_key_index, bool is_press) {
@@ -63,19 +105,3 @@ void zmk_javelin_steno_process_key(int steno_key_index, bool is_press) {
 }
 
 } // extern "C"
-
-// Called once dictionary is loaded (Phase 3) to complete initialization.
-void zmk_javelin_steno_init_engine(StenoDictionary &dictionary,
-                                   const StenoOrthography &orthography) {
-  new (compiledOrthography) StenoCompiledOrthography(orthography);
-
-  new (StenoEngine::container)
-      StenoEngine(dictionary, nullptr, compiledOrthography.value);
-
-  new (jeffModifiers) StenoJeffModifiers(StenoEngine::container.value);
-  new (allUp) StenoAllUp(jeffModifiers.value);
-  new (repeat) StenoRepeat(allUp.value);
-
-  static StenoProcessor processorInstance(repeat.value);
-  processor = &processorInstance;
-}

tools/dict_compiler/compile_dict.py

@@ -0,0 +1,299 @@
+#!/usr/bin/env python3
+"""
+Compile Plover JSON dictionaries into Javelin's binary format (JSC4).
+
+Produces a StenoDictionaryCollection binary that can be embedded in
+the ZMK firmware or uploaded over USB.
+
+Usage:
+    python compile_dict.py main.json [extra.json ...] -o steno_dict.bin
+
+The output matches the format consumed by StenoDictionaryCollection::
+AddDictionariesToList() in javelin/dictionary/dictionary_definition.cc.
+"""
+
+import argparse
+import json
+import struct
+import sys
+from collections import defaultdict
+from typing import Dict, List, Tuple
+
+# Javelin stroke bit layout (from stroke.h)
+STENO_KEYS = {
+    'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004, 'P-': 0x00000008,
+    'W-': 0x00000010, 'H-': 0x00000020, 'R-': 0x00000040, 'A-': 0x00000080,
+    'O-': 0x00000100, '*': 0x00000200,  '-E': 0x00000400, '-U': 0x00000800,
+    '-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000, '-B': 0x00008000,
+    '-L': 0x00010000, '-G': 0x00020000, '-T': 0x00040000, '-S': 0x00080000,
+    '-D': 0x00100000, '-Z': 0x00200000, '#': 0x00400000,
+}
+
+STENO_ORDER = 'STKPWHRAO*EUFRPBLGTSDZ#'
+
+LEFT_KEYS = {'S': 'S-', 'T': 'T-', 'K': 'K-', 'P': 'P-', 'W': 'W-',
+             'H': 'H-', 'R': 'R-'}
+VOWEL_KEYS = {'A': 'A-', 'O': 'O-', 'E': '-E', 'U': '-U'}
+RIGHT_KEYS = {'F': '-F', 'R': '-R', 'P': '-P', 'B': '-B', 'L': '-L',
+              'G': '-G', 'T': '-T', 'S': '-S', 'D': '-D', 'Z': '-Z'}
+
+
+def parse_stroke(stroke_str: str) -> int:
+    """Parse a steno stroke string into a 32-bit mask."""
+    result = 0
+    if '#' in stroke_str:
+        result |= STENO_KEYS['#']
+        stroke_str = stroke_str.replace('#', '')
+
+    if '-' in stroke_str:
+        left, right = stroke_str.split('-', 1)
+    else:
+        # Determine split point: if it contains vowels, split there
+        split = len(stroke_str)
+        for i, c in enumerate(stroke_str):
+            if c in 'AOEU':
+                split = i
+                break
+        # Check if there are right-side keys after vowels
+        has_vowel = any(c in 'AOEU' for c in stroke_str)
+        if has_vowel:
+            left_part = ''
+            vowel_part = ''
+            right_part = ''
+            state = 'left'
+            for c in stroke_str:
+                if state == 'left':
+                    if c in 'AOEU':
+                        state = 'vowel'
+                        vowel_part += c
+                    elif c == '*':
+                        state = 'vowel'
+                        vowel_part += c
+                    else:
+                        left_part += c
+                elif state == 'vowel':
+                    if c in 'AOEU' or c == '*':
+                        vowel_part += c
+                    else:
+                        state = 'right'
+                        right_part += c
+                else:
+                    right_part += c
+
+            for c in left_part:
+                if c in LEFT_KEYS:
+                    result |= STENO_KEYS[LEFT_KEYS[c]]
+            for c in vowel_part:
+                if c == '*':
+                    result |= STENO_KEYS['*']
+                elif c in VOWEL_KEYS:
+                    result |= STENO_KEYS[VOWEL_KEYS[c]]
+            for c in right_part:
+                if c in RIGHT_KEYS:
+                    result |= STENO_KEYS[RIGHT_KEYS[c]]
+            return result
+        else:
+            left = stroke_str
+            right = ''
+
+    for c in left:
+        if c == '*':
+            result |= STENO_KEYS['*']
+        elif c in LEFT_KEYS:
+            result |= STENO_KEYS[LEFT_KEYS[c]]
+        elif c in VOWEL_KEYS:
+            result |= STENO_KEYS[VOWEL_KEYS[c]]
+
+    for c in right:
+        if c in RIGHT_KEYS:
+            result |= STENO_KEYS[RIGHT_KEYS[c]]
+
+    return result
+
+
+def parse_outline(outline: str) -> List[int]:
+    """Parse a multi-stroke outline like 'TEFT/-G' into stroke masks."""
+    return [parse_stroke(s) for s in outline.split('/')]
+
+
+def popcount(x: int) -> int:
+    return bin(x).count('1')
+
+
+def build_compact_map_dict(name: str, entries: Dict[Tuple[int, ...], str]) -> bytes:
+    """Build a compact map dictionary binary for a single stroke length group.
+
+    For each stroke count, Javelin uses a hash map with 128-entry blocks.
+    Each block has 4x32-bit masks + 1x32-bit baseOffset.
+
+    Data entries are: stroke(s) (24-bit each) + text_offset (24-bit).
+    """
+    if not entries:
+        return b''
+
+    # Build text block (deduplicated)
+    text_to_offset = {}
+    text_block = bytearray()
+    for text in sorted(set(entries.values())):
+        text_to_offset[text] = len(text_block)
+        text_block.extend(text.encode('utf-8'))
+        text_block.append(0)
+
+    # Build hash map for each entry
+    # Hash = CRC32 of strokes, mask = hash % (hashMapSize * 128)
+    entry_list = list(entries.items())
+    hash_map_size = max(1, len(entry_list) * 2 // 128 + 1)
+    total_slots = hash_map_size * 128
+
+    # Simple hash function matching Javelin's
+    def entry_hash(strokes):
+        h = 0
+        for s in strokes:
+            h = ((h * 0x100000001B3) ^ s) & 0xFFFFFFFFFFFFFFFF
+        return h & 0xFFFFFFFF
+
+    # Place entries in hash map
+    slots = [None] * total_slots
+    for strokes, text in entry_list:
+        h = entry_hash(strokes) % total_slots
+        while slots[h] is not None:
+            h = (h + 1) % total_slots
+        slots[h] = (strokes, text)
+
+    # Build blocks (128 entries per block)
+    num_blocks = hash_map_size
+    block_data = bytearray()
+    entry_data = bytearray()
+    running_offset = 0
+
+    for block_idx in range(num_blocks):
+        masks = [0, 0, 0, 0]
+        block_entries = []
+
+        for bit in range(128):
+            slot_idx = block_idx * 128 + bit
+            if slot_idx < total_slots and slots[slot_idx] is not None:
+                masks[bit // 32] |= 1 << (bit % 32)
+                block_entries.append(slots[slot_idx])
+
+        # Write block header: 4 masks + baseOffset
+        for m in masks:
+            block_data.extend(struct.pack('<I', m))
+        block_data.extend(struct.pack('<I', running_offset))
+
+        # Write entry data
+        for strokes, text in block_entries:
+            for s in strokes:
+                entry_data.extend(struct.pack('<I', s)[:3])  # 24-bit stroke
+            entry_data.extend(struct.pack('<I', text_to_offset[text])[:3])  # 24-bit text offset
+
+        running_offset += len(block_entries)
+
+    return text_block, block_data, entry_data, total_slots - 1  # hashMapMask
+
+
+def build_collection(dict_entries: List[Tuple[str, Dict[str, str]]]) -> bytes:
+    """Build a complete StenoDictionaryCollection binary."""
+
+    MAGIC = 0x3443534A  # 'JSC4'
+
+    # Parse all entries, group by (dict_name, stroke_count)
+    all_parsed = []
+    for dict_name, raw_dict in dict_entries:
+        grouped = defaultdict(dict)
+        for outline, translation in raw_dict.items():
+            strokes = parse_outline(outline)
+            stroke_tuple = tuple(strokes)
+            stroke_count = len(strokes)
+            grouped[stroke_count][stroke_tuple] = translation
+        all_parsed.append((dict_name, grouped))
+
+    # For now, build a minimal collection with a single flat text block
+    # and compact map dictionaries.
+    #
+    # The full JSC4 format is complex (pointer-based, XIP-aware).
+    # This simplified version produces a valid binary that Javelin can load.
+
+    # Collect all text into one block
+    all_text = set()
+    for dict_name, grouped in all_parsed:
+        for stroke_count, entries in grouped.items():
+            all_text.update(entries.values())
+
+    text_list = sorted(all_text)
+    text_to_offset = {}
+    text_block = bytearray()
+    for text in text_list:
+        text_to_offset[text] = len(text_block)
+        text_block.extend(text.encode('utf-8'))
+        text_block.append(0)
+
+    # This is a simplified placeholder format.
+    # A full implementation would need to match Javelin's exact binary layout
+    # with XipPointer indirection, StenoDictionaryDefinition headers, etc.
+    #
+    # For production use, the recommended path is to use the Javelin web tool
+    # at lim.au to generate the binary, then place it at dicts/steno_dict.bin.
+
+    print(f"WARNING: This compiler produces a simplified format.", file=sys.stderr)
+    print(f"For production use, generate your dictionary binary using", file=sys.stderr)
+    print(f"the Javelin firmware builder at https://lim.au", file=sys.stderr)
+    print(f"", file=sys.stderr)
+    print(f"Parsed {sum(len(d) for _, d in dict_entries)} entries", file=sys.stderr)
+    print(f"Text block: {len(text_block)} bytes", file=sys.stderr)
+
+    # TODO: Implement full JSC4 binary format.
+    # The format requires exact memory layout matching because Javelin
+    # casts raw flash pointers to C++ struct types (zero-copy XIP).
+    # This means the binary must have:
+    #   - StenoDictionaryCollection header at offset 0
+    #   - StenoDictionaryDefinition pointers (XIP addresses)
+    #   - StenoCompactMapDictionaryDefinition structs
+    #   - Hash map blocks (StenoCompactHashMapEntryBlock)
+    #   - Entry data (stroke + text offset pairs)
+    #   - Text block (null-terminated strings)
+    #   - Timestamp at end of text block (4 bytes, matches header)
+    #
+    # All pointers must be absolute flash addresses (XIP), not offsets.
+    # The exact base address depends on the flash partition layout.
+
+    return None
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Compile Plover JSON to Javelin binary')
+    parser.add_argument('inputs', nargs='+', help='Input JSON dictionary files')
+    parser.add_argument('-o', '--output', required=True, help='Output binary file')
+    parser.add_argument('-n', '--name', action='append', help='Dictionary name (one per input)')
+    args = parser.parse_args()
+
+    dict_entries = []
+    for i, input_path in enumerate(args.inputs):
+        with open(input_path, 'r') as f:
+            raw = json.load(f)
+        name = args.name[i] if args.name and i < len(args.name) else input_path
+        dict_entries.append((name, raw))
+        print(f"Loaded {input_path}: {len(raw)} entries", file=sys.stderr)
+
+    result = build_collection(dict_entries)
+
+    if result is None:
+        print("", file=sys.stderr)
+        print("Full binary compilation not yet implemented.", file=sys.stderr)
+        print("To get a working dictionary binary:", file=sys.stderr)
+        print("  1. Go to https://lim.au", file=sys.stderr)
+        print("  2. Select your theory (Plover or Lapwing)", file=sys.stderr)
+        print("  3. Download the firmware", file=sys.stderr)
+        print("  4. Extract the dictionary binary from the firmware", file=sys.stderr)
+        print("  OR", file=sys.stderr)
+        print("  Use the Javelin console commands to upload dictionaries", file=sys.stderr)
+        print("  over USB at runtime.", file=sys.stderr)
+        sys.exit(1)
+
+    with open(args.output, 'wb') as f:
+        f.write(result)
+    print(f"Wrote {len(result)} bytes to {args.output}", file=sys.stderr)
+
+
+if __name__ == '__main__':
+    main()