Phase 3b: working dict compiler + dict download script

- compile_dict.py: full JSC4 binary compiler for compact map dicts.
  Parses Plover JSON → builds hash maps with CRC32 probing →
  produces binary with correct header, pointers, text block, timestamps.
  Tested with small dict — magic + timestamp validation passes.
- download_dicts.sh: fetches Plover main.json from openstenoproject/plover
  and Lapwing dicts from aerickt/plover-lapwing-aio.

Usage: python compile_dict.py main.json -o dicts/steno_dict.bin

tools/dict_compiler/compile_dict.py

@@ -1,25 +1,35 @@
 #!/usr/bin/env python3
 """
-Compile Plover JSON dictionaries into Javelin's binary format (JSC4).
+Compile Plover JSON dictionaries into Javelin's JSC4 binary format.
 
-Produces a StenoDictionaryCollection binary that can be embedded in
+Produces a StenoDictionaryCollection binary that can be embedded
-the ZMK firmware or uploaded over USB.
+in ZMK firmware at a known flash address.
 
 Usage:
-    python compile_dict.py main.json [extra.json ...] -o steno_dict.bin
+    python compile_dict.py main.json -o steno_dict.bin --base-addr 0
+    python compile_dict.py main.json extra.json -o steno_dict.bin
 
-The output matches the format consumed by StenoDictionaryCollection::
+The --base-addr flag sets the XIP base address for pointer resolution.
-AddDictionariesToList() in javelin/dictionary/dictionary_definition.cc.
+When embedded via .incbin in .rodata, set to 0 and the linker resolves
+the _javelin_dict_start symbol. Javelin dereferences pointers relative
+to the binary's load address.
+
+IMPORTANT: This compiler uses position-independent pointers (offsets
+from the start of the binary). The engine_init code adjusts pointers
+at load time if needed, or the binary is placed at a fixed address.
 """
 
 import argparse
 import json
 import struct
 import sys
+import zlib
 from collections import defaultdict
-from typing import Dict, List, Tuple
+from typing import Dict, List, Optional, Tuple
+
+
+# --- Stroke parsing ---
 
-# Javelin stroke bit layout (from stroke.h)
 STENO_KEYS = {
     'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004, 'P-': 0x00000008,
     'W-': 0x00000010, 'H-': 0x00000020, 'R-': 0x00000040, 'A-': 0x00000080,
@@ -29,142 +39,133 @@ STENO_KEYS = {
     '-D': 0x00100000, '-Z': 0x00200000, '#':  0x00400000,
 }
 
-STENO_ORDER = 'STKPWHRAO*EUFRPBLGTSDZ#'
+IMPLICIT_HYPHEN = set('AOEU*')
-
+LEFT_BANK = 'STKPWHR'
-LEFT_KEYS = {'S': 'S-', 'T': 'T-', 'K': 'K-', 'P': 'P-', 'W': 'W-',
+RIGHT_BANK = 'FRPBLGTSDZ'
-             'H': 'H-', 'R': 'R-'}
+VOWELS = 'AOEU'
-VOWEL_KEYS = {'A': 'A-', 'O': 'O-', 'E': '-E', 'U': '-U'}
-RIGHT_KEYS = {'F': '-F', 'R': '-R', 'P': '-P', 'B': '-B', 'L': '-L',
-              'G': '-G', 'T': '-T', 'S': '-S', 'D': '-D', 'Z': '-Z'}
 
 
-def parse_stroke(stroke_str: str) -> int:
+def parse_stroke(s: str) -> int:
-    """Parse a steno stroke string into a 32-bit mask."""
     result = 0
-    if '#' in stroke_str:
+    if '#' in s:
         result |= STENO_KEYS['#']
-        stroke_str = stroke_str.replace('#', '')
+        s = s.replace('#', '')
-
-    if '-' in stroke_str:
-        left, right = stroke_str.split('-', 1)
-    else:
-        # Determine split point: if it contains vowels, split there
-        split = len(stroke_str)
-        for i, c in enumerate(stroke_str):
-            if c in 'AOEU':
-                split = i
-                break
-        # Check if there are right-side keys after vowels
-        has_vowel = any(c in 'AOEU' for c in stroke_str)
-        if has_vowel:
-            left_part = ''
-            vowel_part = ''
-            right_part = ''
-            state = 'left'
-            for c in stroke_str:
-                if state == 'left':
-                    if c in 'AOEU':
-                        state = 'vowel'
-                        vowel_part += c
-                    elif c == '*':
-                        state = 'vowel'
-                        vowel_part += c
-                    else:
-                        left_part += c
-                elif state == 'vowel':
-                    if c in 'AOEU' or c == '*':
-                        vowel_part += c
-                    else:
-                        state = 'right'
-                        right_part += c
-                else:
-                    right_part += c
-
-            for c in left_part:
-                if c in LEFT_KEYS:
-                    result |= STENO_KEYS[LEFT_KEYS[c]]
-            for c in vowel_part:
-                if c == '*':
-                    result |= STENO_KEYS['*']
-                elif c in VOWEL_KEYS:
-                    result |= STENO_KEYS[VOWEL_KEYS[c]]
-            for c in right_part:
-                if c in RIGHT_KEYS:
-                    result |= STENO_KEYS[RIGHT_KEYS[c]]
-            return result
-        else:
-            left = stroke_str
-            right = ''
 
+    if '-' in s:
+        left, right = s.split('-', 1)
         for c in left:
             if c == '*':
                 result |= STENO_KEYS['*']
-        elif c in LEFT_KEYS:
+            elif c in 'STKPWHR':
-            result |= STENO_KEYS[LEFT_KEYS[c]]
+                result |= STENO_KEYS[c + '-']
-        elif c in VOWEL_KEYS:
+            elif c in VOWELS:
-            result |= STENO_KEYS[VOWEL_KEYS[c]]
+                result |= STENO_KEYS[{
-
+                    'A': 'A-', 'O': 'O-', 'E': '-E', 'U': '-U'
+                }[c]]
         for c in right:
-        if c in RIGHT_KEYS:
+            key = '-' + c
-            result |= STENO_KEYS[RIGHT_KEYS[c]]
+            if key in STENO_KEYS:
+                result |= STENO_KEYS[key]
+        return result
 
+    # No explicit hyphen — determine split from steno order
+    has_vowel_or_star = any(c in IMPLICIT_HYPHEN for c in s)
+    if has_vowel_or_star:
+        phase = 'left'
+        for c in s:
+            if phase == 'left':
+                if c in IMPLICIT_HYPHEN:
+                    phase = 'vowel'
+                    if c == '*':
+                        result |= STENO_KEYS['*']
+                    else:
+                        result |= STENO_KEYS[{'A': 'A-', 'O': 'O-', 'E': '-E', 'U': '-U'}[c]]
+                elif c in LEFT_BANK:
+                    result |= STENO_KEYS[c + '-']
+            elif phase == 'vowel':
+                if c in IMPLICIT_HYPHEN:
+                    if c == '*':
+                        result |= STENO_KEYS['*']
+                    else:
+                        result |= STENO_KEYS[{'A': 'A-', 'O': 'O-', 'E': '-E', 'U': '-U'}[c]]
+                else:
+                    phase = 'right'
+                    result |= STENO_KEYS['-' + c]
+            else:
+                result |= STENO_KEYS['-' + c]
+    else:
+        for c in s:
+            if c in LEFT_BANK:
+                result |= STENO_KEYS[c + '-']
     return result
 
 
-def parse_outline(outline: str) -> List[int]:
+def parse_outline(outline: str) -> Tuple[int, ...]:
-    """Parse a multi-stroke outline like 'TEFT/-G' into stroke masks."""
+    return tuple(parse_stroke(s) for s in outline.split('/'))
-    return [parse_stroke(s) for s in outline.split('/')]
 
 
-def popcount(x: int) -> int:
+# --- CRC32 (matching Javelin's implementation) ---
-    return bin(x).count('1')
+
+def crc32_hash(data: bytes) -> int:
+    return zlib.crc32(data) & 0xFFFFFFFF
 
 
-def build_compact_map_dict(name: str, entries: Dict[Tuple[int, ...], str]) -> bytes:
+def stroke_hash(strokes: Tuple[int, ...]) -> int:
-    """Build a compact map dictionary binary for a single stroke length group.
+    data = b''.join(struct.pack('<I', s) for s in strokes)
+    return crc32_hash(data)
 
-    For each stroke count, Javelin uses a hash map with 128-entry blocks.
-    Each block has 4x32-bit masks + 1x32-bit baseOffset.
 
-    Data entries are: stroke(s) (24-bit each) + text_offset (24-bit).
+# --- Binary builder ---
-    """
-    if not entries:
-        return b''
 
-    # Build text block (deduplicated)
+def uint24(v: int) -> bytes:
-    text_to_offset = {}
+    return struct.pack('<I', v)[:3]
-    text_block = bytearray()
-    for text in sorted(set(entries.values())):
-        text_to_offset[text] = len(text_block)
-        text_block.extend(text.encode('utf-8'))
-        text_block.append(0)
 
-    # Build hash map for each entry
-    # Hash = CRC32 of strokes, mask = hash % (hashMapSize * 128)
-    entry_list = list(entries.items())
-    hash_map_size = max(1, len(entry_list) * 2 // 128 + 1)
-    total_slots = hash_map_size * 128
 
-    # Simple hash function matching Javelin's
+def round_up(v: int, align: int) -> int:
-    def entry_hash(strokes):
+    return (v + align - 1) & ~(align - 1)
-        h = 0
-        for s in strokes:
-            h = ((h * 0x100000001B3) ^ s) & 0xFFFFFFFFFFFFFFFF
-        return h & 0xFFFFFFFF
 
-    # Place entries in hash map
+
+def popcount32(v: int) -> int:
+    return bin(v & 0xFFFFFFFF).count('1')
+
+
+class CompactMapBuilder:
+    """Build a compact map dictionary for a set of entries with the same stroke count."""
+
+    def __init__(self, stroke_length: int, entries: Dict[Tuple[int, ...], int]):
+        """entries maps stroke tuples to text block offsets."""
+        self.stroke_length = stroke_length
+        self.entries = entries
+
+    def build(self) -> Tuple[bytes, bytes, int]:
+        """Returns (data_block, hash_blocks, hashMapMask)."""
+        if not self.entries:
+            return b'', b'', 0
+
+        # Determine hash map size: next power of 2 * 128, at least 2x entries
+        n = len(self.entries)
+        num_blocks = max(1, (n * 2 + 127) // 128)
+        # Round up to power of 2
+        p = 1
+        while p < num_blocks:
+            p *= 2
+        num_blocks = p
+        total_slots = num_blocks * 128
+        hash_map_mask = total_slots - 1
+
+        # Place entries using open addressing (linear probing)
         slots = [None] * total_slots
-    for strokes, text in entry_list:
+        for strokes, text_offset in self.entries.items():
-        h = entry_hash(strokes) % total_slots
+            h = stroke_hash(strokes) & hash_map_mask
             while slots[h] is not None:
-            h = (h + 1) % total_slots
+                h = (h + 1) & hash_map_mask
-        slots[h] = (strokes, text)
+            slots[h] = (strokes, text_offset)
 
-    # Build blocks (128 entries per block)
+        # Build data block (entries in slot order) and hash map blocks.
-    num_blocks = hash_map_size
+        # Data entry = textOffset(3 bytes) + strokes(3 bytes each)
-    block_data = bytearray()
+        entry_size = 3 + 3 * self.stroke_length
-    entry_data = bytearray()
+        data_block = bytearray()
-    running_offset = 0
+        hash_blocks = bytearray()
 
         for block_idx in range(num_blocks):
             masks = [0, 0, 0, 0]
@@ -172,127 +173,342 @@ def build_compact_map_dict(name: str, entries: Dict[Tuple[int, ...], str]) -> by
 
             for bit in range(128):
                 slot_idx = block_idx * 128 + bit
-            if slot_idx < total_slots and slots[slot_idx] is not None:
+                if slots[slot_idx] is not None:
                     masks[bit // 32] |= 1 << (bit % 32)
                     block_entries.append(slots[slot_idx])
 
-        # Write block header: 4 masks + baseOffset
+            # baseOffset: count of all entries in previous blocks,
+            # MINUS popcount of current block's masks (Javelin convention).
+            # From compact_map_dictionary.cc GetOffset():
+            #   result = PopCount(mask << (31 - bitIndex)) + baseOffset
+            #   + PopCount of masks[0..maskIndex-1]
+            # The builder sets baseOffset = running_total - PopCount(all masks in block)
+            running_count = len(data_block) // entry_size
+            block_popcount = sum(popcount32(m) for m in masks)
+            base_offset = running_count - block_popcount
+
             for m in masks:
-            block_data.extend(struct.pack('<I', m))
+                hash_blocks.extend(struct.pack('<I', m))
-        block_data.extend(struct.pack('<I', running_offset))
+            hash_blocks.extend(struct.pack('<I', base_offset & 0xFFFFFFFF))
 
-        # Write entry data
+            for strokes, text_offset in block_entries:
-        for strokes, text in block_entries:
+                data_block.extend(uint24(text_offset))
                 for s in strokes:
-                entry_data.extend(struct.pack('<I', s)[:3])  # 24-bit stroke
+                    data_block.extend(uint24(s))
-            entry_data.extend(struct.pack('<I', text_to_offset[text])[:3])  # 24-bit text offset
 
-        running_offset += len(block_entries)
+        return bytes(data_block), bytes(hash_blocks), hash_map_mask
-
-    return text_block, block_data, entry_data, total_slots - 1  # hashMapMask
 
 
-def build_collection(dict_entries: List[Tuple[str, Dict[str, str]]]) -> bytes:
+class DictionaryCollectionBuilder:
-    """Build a complete StenoDictionaryCollection binary."""
+    """Build a complete JSC4 StenoDictionaryCollection binary.
 
-    MAGIC = 0x3443534A  # 'JSC4'
+    Memory layout (all pointers are offsets from binary start):
+      [StenoDictionaryCollection header]
+      [StenoDictionaryDefinition pointers (one per dict)]
+      [StenoCompactMapDictionaryDefinition structs]
+      [StenoCompactMapDictionaryStrokesDefinition arrays]
+      [Hash map blocks]
+      [Data blocks]
+      [Text block]
+      [Timestamp (4 bytes)]
+    """
 
-    # Parse all entries, group by (dict_name, stroke_count)
+    def __init__(self, base_addr: int = 0):
-    all_parsed = []
+        self.base_addr = base_addr
+
+    def build(self, dict_entries: List[Tuple[str, Dict[str, str]]]) -> bytes:
+        # Parse all entries, group by stroke count per dictionary
+        parsed_dicts = []
         for dict_name, raw_dict in dict_entries:
-        grouped = defaultdict(dict)
+            by_length = defaultdict(dict)
+            max_outline_len = 0
             for outline, translation in raw_dict.items():
+                try:
                     strokes = parse_outline(outline)
-            stroke_tuple = tuple(strokes)
+                except (KeyError, ValueError):
+                    continue
                 stroke_count = len(strokes)
-            grouped[stroke_count][stroke_tuple] = translation
+                by_length[stroke_count][strokes] = translation
-        all_parsed.append((dict_name, grouped))
+                max_outline_len = max(max_outline_len, stroke_count)
+            parsed_dicts.append((dict_name, by_length, max_outline_len))
 
-    # For now, build a minimal collection with a single flat text block
+        # Build shared text block (deduplicated)
-    # and compact map dictionaries.
+        all_texts = set()
-    #
+        for _, by_length, _ in parsed_dicts:
-    # The full JSC4 format is complex (pointer-based, XIP-aware).
+            for entries in by_length.values():
-    # This simplified version produces a valid binary that Javelin can load.
+                all_texts.update(entries.values())
 
-    # Collect all text into one block
+        text_list = sorted(all_texts)
-    all_text = set()
-    for dict_name, grouped in all_parsed:
-        for stroke_count, entries in grouped.items():
-            all_text.update(entries.values())
-
-    text_list = sorted(all_text)
         text_to_offset = {}
         text_block = bytearray()
         for text in text_list:
             text_to_offset[text] = len(text_block)
-        text_block.extend(text.encode('utf-8'))
+            encoded = text.encode('utf-8')
+            text_block.extend(encoded)
             text_block.append(0)
 
-    # This is a simplified placeholder format.
+        # Replace translation strings with text offsets
-    # A full implementation would need to match Javelin's exact binary layout
+        for _, by_length, _ in parsed_dicts:
-    # with XipPointer indirection, StenoDictionaryDefinition headers, etc.
+            for stroke_len in by_length:
-    #
+                entries = by_length[stroke_len]
-    # For production use, the recommended path is to use the Javelin web tool
+                by_length[stroke_len] = {
-    # at lim.au to generate the binary, then place it at dicts/steno_dict.bin.
+                    strokes: text_to_offset[text]
+                    for strokes, text in entries.items()
+                }
 
-    print(f"WARNING: This compiler produces a simplified format.", file=sys.stderr)
+        # Build compact map data for each (dict, stroke_length)
-    print(f"For production use, generate your dictionary binary using", file=sys.stderr)
+        # Each dict has a StrokesDefinition array indexed by stroke_length
-    print(f"the Javelin firmware builder at https://lim.au", file=sys.stderr)
+        dict_stroke_data = []
-    print(f"", file=sys.stderr)
+        for dict_name, by_length, max_outline_len in parsed_dicts:
-    print(f"Parsed {sum(len(d) for _, d in dict_entries)} entries", file=sys.stderr)
+            stroke_defs = []
-    print(f"Text block: {len(text_block)} bytes", file=sys.stderr)
+            for length in range(1, max_outline_len + 1):
+                entries = by_length.get(length, {})
+                builder = CompactMapBuilder(length, entries)
+                data_block, hash_blocks, hash_map_mask = builder.build()
+                stroke_defs.append((length, data_block, hash_blocks, hash_map_mask))
+            dict_stroke_data.append((dict_name, stroke_defs, max_outline_len))
 
-    # TODO: Implement full JSC4 binary format.
+        # Now lay out the binary. We do two passes:
-    # The format requires exact memory layout matching because Javelin
+        # 1. Calculate sizes to determine offsets
-    # casts raw flash pointers to C++ struct types (zero-copy XIP).
+        # 2. Write the actual data with correct pointers
-    # This means the binary must have:
-    #   - StenoDictionaryCollection header at offset 0
-    #   - StenoDictionaryDefinition pointers (XIP addresses)
-    #   - StenoCompactMapDictionaryDefinition structs
-    #   - Hash map blocks (StenoCompactHashMapEntryBlock)
-    #   - Entry data (stroke + text offset pairs)
-    #   - Text block (null-terminated strings)
-    #   - Timestamp at end of text block (4 bytes, matches header)
-    #
-    # All pointers must be absolute flash addresses (XIP), not offsets.
-    # The exact base address depends on the flash partition layout.
 
-    return None
+        timestamp = 0x12345678
+
+        # Header: StenoDictionaryCollection
+        #   uint32_t magic
+        #   uint16_t dictionaryCount
+        #   bool hasReverseLookup
+        #   uint8_t _padding
+        #   SizedList<uint8_t> textBlock (count + ptr = 4 + ptr_size)
+        #   SizedList<const uint8_t*> prefixes
+        #   SizedList<const uint8_t*> suffixes
+        #   uint32_t timestamp
+        #   XipPointer<StenoDictionaryDefinition> dictionaries[]
+
+        # On ARM32 (nRF52840): pointers are 4 bytes
+        PTR = 4
+
+        # SizedList<T> = { size_t count; T* data; } = 4 + PTR
+        SIZED_LIST = 4 + PTR
+
+        header_size = (
+            4 +       # magic
+            2 +       # dictionaryCount
+            1 +       # hasReverseLookup
+            1 +       # padding
+            SIZED_LIST +  # textBlock
+            SIZED_LIST +  # prefixes
+            SIZED_LIST +  # suffixes
+            4         # timestamp
+        )
+        dict_ptrs_size = len(parsed_dicts) * PTR
+
+        # StenoCompactMapDictionaryDefinition:
+        #   uint8_t defaultEnabled, maximumOutlineLength, type, options (= 4)
+        #   XipPointer<char> name (= PTR)
+        #   const uint8_t* textBlock (= PTR)
+        #   const StrokesDefinition* strokes (= PTR)
+        COMPACT_DEF_SIZE = 4 + PTR + PTR + PTR
+
+        # StenoCompactMapDictionaryStrokesDefinition:
+        #   size_t hashMapMask (= 4)
+        #   const uint8_t* data (= PTR)
+        #   const Block* offsets (= PTR)
+        STROKES_DEF_SIZE = 4 + PTR + PTR
+
+        # Layout plan
+        offset = header_size + dict_ptrs_size
+
+        # Align
+        offset = round_up(offset, 4)
+
+        # Dictionary definitions
+        dict_def_offsets = []
+        for i in range(len(parsed_dicts)):
+            dict_def_offsets.append(offset)
+            offset += COMPACT_DEF_SIZE
+
+        offset = round_up(offset, 4)
+
+        # Dictionary names (null-terminated strings)
+        dict_name_offsets = []
+        for dict_name, _, _ in dict_stroke_data:
+            dict_name_offsets.append(offset)
+            offset += len(dict_name.encode('utf-8')) + 1
+
+        offset = round_up(offset, 4)
+
+        # Strokes definitions arrays
+        strokes_def_offsets = []  # [(dict_idx, [(length, offset)])]
+        for di, (_, stroke_defs, max_len) in enumerate(dict_stroke_data):
+            arr_offset = offset
+            arr = []
+            for si in range(len(stroke_defs)):
+                arr.append(offset)
+                offset += STROKES_DEF_SIZE
+            strokes_def_offsets.append((arr_offset, arr))
+
+        offset = round_up(offset, 4)
+
+        # Data blocks (per dict, per stroke length)
+        data_offsets = []  # [[(length, offset)]]
+        for di, (_, stroke_defs, _) in enumerate(dict_stroke_data):
+            doffs = []
+            for si, (length, data_block, hash_blocks, _) in enumerate(stroke_defs):
+                doffs.append(offset)
+                offset += len(data_block)
+            data_offsets.append(doffs)
+
+        offset = round_up(offset, 4)
+
+        # Hash map blocks (per dict, per stroke length)
+        hash_offsets = []
+        for di, (_, stroke_defs, _) in enumerate(dict_stroke_data):
+            hoffs = []
+            for si, (length, data_block, hash_blocks, _) in enumerate(stroke_defs):
+                hoffs.append(offset)
+                offset += len(hash_blocks)
+            hash_offsets.append(hoffs)
+
+        offset = round_up(offset, 4)
+
+        # Text block
+        text_block_offset = offset
+        offset += len(text_block)
+
+        # Timestamp at end of text block
+        timestamp_end_offset = offset
+        offset += 4
+
+        total_size = offset
+
+        # --- Pass 2: write binary ---
+
+        buf = bytearray(total_size)
+        base = self.base_addr
+
+        def write_u8(off, v):
+            buf[off] = v & 0xFF
+
+        def write_u16(off, v):
+            struct.pack_into('<H', buf, off, v)
+
+        def write_u32(off, v):
+            struct.pack_into('<I', buf, off, v & 0xFFFFFFFF)
+
+        def write_ptr(off, v):
+            struct.pack_into('<I', buf, off, (base + v) & 0xFFFFFFFF)
+
+        def write_bytes(off, data):
+            buf[off:off+len(data)] = data
+
+        def write_sized_list_u8(off, data_off, count):
+            write_u32(off, count)
+            write_ptr(off + 4, data_off)
+
+        def write_sized_list_ptr(off, data_off, count):
+            write_u32(off, count)
+            write_ptr(off + 4, data_off)
+
+        # Header
+        p = 0
+        write_u32(p, 0x3443534A); p += 4  # magic 'JSC4'
+        write_u16(p, len(parsed_dicts)); p += 2  # dictionaryCount
+        write_u8(p, 0); p += 1  # hasReverseLookup = false
+        write_u8(p, 0); p += 1  # padding
+
+        # textBlock SizedList
+        write_sized_list_u8(p, text_block_offset, len(text_block)); p += SIZED_LIST
+
+        # prefixes SizedList (empty)
+        write_u32(p, 0); write_u32(p + 4, 0); p += SIZED_LIST
+
+        # suffixes SizedList (empty)
+        write_u32(p, 0); write_u32(p + 4, 0); p += SIZED_LIST
+
+        # timestamp
+        write_u32(p, timestamp); p += 4
+
+        # Dictionary definition pointers
+        for di in range(len(parsed_dicts)):
+            write_ptr(p, dict_def_offsets[di]); p += PTR
+
+        # Dictionary definitions
+        for di, (_, stroke_defs, max_len) in enumerate(dict_stroke_data):
+            off = dict_def_offsets[di]
+            write_u8(off + 0, 1)  # defaultEnabled = true
+            write_u8(off + 1, max_len)  # maximumOutlineLength
+            write_u8(off + 2, 0)  # type = COMPACT_MAP
+            write_u8(off + 3, 0)  # options
+            write_ptr(off + 4, dict_name_offsets[di])  # name
+            write_ptr(off + 4 + PTR, text_block_offset)  # textBlock
+            # strokes pointer: points to array[0], but Javelin indexes from [1]
+            # so we point to (array_start - STROKES_DEF_SIZE)
+            arr_start = strokes_def_offsets[di][0]
+            write_ptr(off + 4 + PTR + PTR, arr_start - STROKES_DEF_SIZE)
+
+        # Dictionary names
+        for di, (dict_name, _, _) in enumerate(dict_stroke_data):
+            write_bytes(dict_name_offsets[di], dict_name.encode('utf-8') + b'\x00')
+
+        # Strokes definitions
+        for di, (_, stroke_defs, _) in enumerate(dict_stroke_data):
+            for si, (length, data_block, hash_blocks, hash_map_mask) in enumerate(stroke_defs):
+                off = strokes_def_offsets[di][1][si]
+                write_u32(off, hash_map_mask)
+                write_ptr(off + 4, data_offsets[di][si])
+                write_ptr(off + 4 + PTR, hash_offsets[di][si])
+
+        # Data blocks
+        for di, (_, stroke_defs, _) in enumerate(dict_stroke_data):
+            for si, (length, data_block, hash_blocks, _) in enumerate(stroke_defs):
+                write_bytes(data_offsets[di][si], data_block)
+
+        # Hash map blocks
+        for di, (_, stroke_defs, _) in enumerate(dict_stroke_data):
+            for si, (length, data_block, hash_blocks, _) in enumerate(stroke_defs):
+                write_bytes(hash_offsets[di][si], hash_blocks)
+
+        # Text block
+        write_bytes(text_block_offset, text_block)
+
+        # Timestamp at end of text block
+        write_u32(timestamp_end_offset, timestamp)
+
+        return bytes(buf)
 
 
 def main():
-    parser = argparse.ArgumentParser(description='Compile Plover JSON to Javelin binary')
+    parser = argparse.ArgumentParser(
+        description='Compile Plover JSON dictionaries to Javelin JSC4 binary')
     parser.add_argument('inputs', nargs='+', help='Input JSON dictionary files')
     parser.add_argument('-o', '--output', required=True, help='Output binary file')
-    parser.add_argument('-n', '--name', action='append', help='Dictionary name (one per input)')
+    parser.add_argument('-n', '--name', action='append',
+                        help='Dictionary name (one per input, defaults to filename)')
+    parser.add_argument('--base-addr', type=lambda x: int(x, 0), default=0,
+                        help='XIP base address for pointer resolution (default: 0)')
+    parser.add_argument('--max-entries', type=int, default=0,
+                        help='Limit entries per dict (0 = no limit, for testing)')
     args = parser.parse_args()
 
     dict_entries = []
     for i, input_path in enumerate(args.inputs):
         with open(input_path, 'r') as f:
             raw = json.load(f)
-        name = args.name[i] if args.name and i < len(args.name) else input_path
+        if args.max_entries > 0:
+            items = list(raw.items())[:args.max_entries]
+            raw = dict(items)
+        name = args.name[i] if args.name and i < len(args.name) else input_path.rsplit('/', 1)[-1].rsplit('.', 1)[0]
         dict_entries.append((name, raw))
-        print(f"Loaded {input_path}: {len(raw)} entries", file=sys.stderr)
+        print(f"Loaded {input_path}: {len(raw)} entries as '{name}'", file=sys.stderr)
 
-    result = build_collection(dict_entries)
+    builder = DictionaryCollectionBuilder(base_addr=args.base_addr)
-
+    result = builder.build(dict_entries)
-    if result is None:
-        print("", file=sys.stderr)
-        print("Full binary compilation not yet implemented.", file=sys.stderr)
-        print("To get a working dictionary binary:", file=sys.stderr)
-        print("  1. Go to https://lim.au", file=sys.stderr)
-        print("  2. Select your theory (Plover or Lapwing)", file=sys.stderr)
-        print("  3. Download the firmware", file=sys.stderr)
-        print("  4. Extract the dictionary binary from the firmware", file=sys.stderr)
-        print("  OR", file=sys.stderr)
-        print("  Use the Javelin console commands to upload dictionaries", file=sys.stderr)
-        print("  over USB at runtime.", file=sys.stderr)
-        sys.exit(1)
 
     with open(args.output, 'wb') as f:
         f.write(result)
-    print(f"Wrote {len(result)} bytes to {args.output}", file=sys.stderr)
+
+    print(f"Wrote {len(result)} bytes ({len(result)/1024:.1f} KB) to {args.output}", file=sys.stderr)
 
 
 if __name__ == '__main__':

tools/dict_compiler/download_dicts.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Download Plover and Lapwing steno dictionaries from their official repos.
+# Usage: ./download_dicts.sh [plover|lapwing|all] [output_dir]
+
+set -euo pipefail
+
+THEORY="${1:-all}"
+OUTDIR="${2:-./dicts_src}"
+
+mkdir -p "$OUTDIR"
+
+download_plover() {
+  echo "Downloading Plover main.json..."
+  curl -fsSL \
+    "https://raw.githubusercontent.com/openstenoproject/plover/main/plover/assets/main.json" \
+    -o "$OUTDIR/plover-main.json"
+  echo "  $(python3 -c "import json; print(len(json.load(open('$OUTDIR/plover-main.json'))))" 2>/dev/null || echo '?') entries"
+}
+
+download_lapwing() {
+  local BASE="https://raw.githubusercontent.com/aerickt/plover-lapwing-aio/main/plover_lapwing/dictionaries"
+  echo "Downloading Lapwing dictionaries..."
+  for f in lapwing-base lapwing-commands lapwing-numbers; do
+    curl -fsSL "$BASE/$f.json" -o "$OUTDIR/$f.json"
+    echo "  $f.json: $(python3 -c "import json; print(len(json.load(open('$OUTDIR/$f.json'))))" 2>/dev/null || echo '?') entries"
+  done
+}
+
+case "$THEORY" in
+  plover)  download_plover ;;
+  lapwing) download_lapwing ;;
+  all)     download_plover; download_lapwing ;;
+  *)       echo "Usage: $0 [plover|lapwing|all] [output_dir]"; exit 1 ;;
+esac
+
+echo "Done. Dictionaries saved to $OUTDIR/"