diff --git a/tools/dict_compiler/compile_dict.py b/tools/dict_compiler/compile_dict.py index a0394c2..4d6e1fe 100644 --- a/tools/dict_compiler/compile_dict.py +++ b/tools/dict_compiler/compile_dict.py @@ -1,298 +1,514 @@ #!/usr/bin/env python3 """ -Compile Plover JSON dictionaries into Javelin's binary format (JSC4). +Compile Plover JSON dictionaries into Javelin's JSC4 binary format. -Produces a StenoDictionaryCollection binary that can be embedded in -the ZMK firmware or uploaded over USB. +Produces a StenoDictionaryCollection binary that can be embedded +in ZMK firmware at a known flash address. Usage: - python compile_dict.py main.json [extra.json ...] -o steno_dict.bin + python compile_dict.py main.json -o steno_dict.bin --base-addr 0 + python compile_dict.py main.json extra.json -o steno_dict.bin -The output matches the format consumed by StenoDictionaryCollection:: -AddDictionariesToList() in javelin/dictionary/dictionary_definition.cc. +The --base-addr flag sets the XIP base address for pointer resolution. +When embedded via .incbin in .rodata, set to 0 and the linker resolves +the _javelin_dict_start symbol. Javelin dereferences pointers relative +to the binary's load address. + +IMPORTANT: This compiler uses position-independent pointers (offsets +from the start of the binary). The engine_init code adjusts pointers +at load time if needed, or the binary is placed at a fixed address. """ import argparse import json import struct import sys +import zlib from collections import defaultdict -from typing import Dict, List, Tuple +from typing import Dict, List, Optional, Tuple + + +# --- Stroke parsing --- -# Javelin stroke bit layout (from stroke.h) STENO_KEYS = { 'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004, 'P-': 0x00000008, 'W-': 0x00000010, 'H-': 0x00000020, 'R-': 0x00000040, 'A-': 0x00000080, - 'O-': 0x00000100, '*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800, + 'O-': 0x00000100, '*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800, '-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000, '-B': 0x00008000, '-L': 0x00010000, '-G': 0x00020000, '-T': 0x00040000, '-S': 0x00080000, - '-D': 0x00100000, '-Z': 0x00200000, '#': 0x00400000, + '-D': 0x00100000, '-Z': 0x00200000, '#': 0x00400000, } -STENO_ORDER = 'STKPWHRAO*EUFRPBLGTSDZ#' - -LEFT_KEYS = {'S': 'S-', 'T': 'T-', 'K': 'K-', 'P': 'P-', 'W': 'W-', - 'H': 'H-', 'R': 'R-'} -VOWEL_KEYS = {'A': 'A-', 'O': 'O-', 'E': '-E', 'U': '-U'} -RIGHT_KEYS = {'F': '-F', 'R': '-R', 'P': '-P', 'B': '-B', 'L': '-L', - 'G': '-G', 'T': '-T', 'S': '-S', 'D': '-D', 'Z': '-Z'} +IMPLICIT_HYPHEN = set('AOEU*') +LEFT_BANK = 'STKPWHR' +RIGHT_BANK = 'FRPBLGTSDZ' +VOWELS = 'AOEU' -def parse_stroke(stroke_str: str) -> int: - """Parse a steno stroke string into a 32-bit mask.""" +def parse_stroke(s: str) -> int: result = 0 - if '#' in stroke_str: + if '#' in s: result |= STENO_KEYS['#'] - stroke_str = stroke_str.replace('#', '') + s = s.replace('#', '') - if '-' in stroke_str: - left, right = stroke_str.split('-', 1) - else: - # Determine split point: if it contains vowels, split there - split = len(stroke_str) - for i, c in enumerate(stroke_str): - if c in 'AOEU': - split = i - break - # Check if there are right-side keys after vowels - has_vowel = any(c in 'AOEU' for c in stroke_str) - if has_vowel: - left_part = '' - vowel_part = '' - right_part = '' - state = 'left' - for c in stroke_str: - if state == 'left': - if c in 'AOEU': - state = 'vowel' - vowel_part += c - elif c == '*': - state = 'vowel' - vowel_part += c + if '-' in s: + left, right = s.split('-', 1) + for c in left: + if c == '*': + result |= STENO_KEYS['*'] + elif c in 'STKPWHR': + result |= STENO_KEYS[c + '-'] + elif c in VOWELS: + result |= STENO_KEYS[{ + 'A': 'A-', 'O': 'O-', 'E': '-E', 'U': '-U' + }[c]] + for c in right: + key = '-' + c + if key in STENO_KEYS: + result |= STENO_KEYS[key] + return result + + # No explicit hyphen — determine split from steno order + has_vowel_or_star = any(c in IMPLICIT_HYPHEN for c in s) + if has_vowel_or_star: + phase = 'left' + for c in s: + if phase == 'left': + if c in IMPLICIT_HYPHEN: + phase = 'vowel' + if c == '*': + result |= STENO_KEYS['*'] else: - left_part += c - elif state == 'vowel': - if c in 'AOEU' or c == '*': - vowel_part += c + result |= STENO_KEYS[{'A': 'A-', 'O': 'O-', 'E': '-E', 'U': '-U'}[c]] + elif c in LEFT_BANK: + result |= STENO_KEYS[c + '-'] + elif phase == 'vowel': + if c in IMPLICIT_HYPHEN: + if c == '*': + result |= STENO_KEYS['*'] else: - state = 'right' - right_part += c + result |= STENO_KEYS[{'A': 'A-', 'O': 'O-', 'E': '-E', 'U': '-U'}[c]] else: - right_part += c - - for c in left_part: - if c in LEFT_KEYS: - result |= STENO_KEYS[LEFT_KEYS[c]] - for c in vowel_part: - if c == '*': - result |= STENO_KEYS['*'] - elif c in VOWEL_KEYS: - result |= STENO_KEYS[VOWEL_KEYS[c]] - for c in right_part: - if c in RIGHT_KEYS: - result |= STENO_KEYS[RIGHT_KEYS[c]] - return result - else: - left = stroke_str - right = '' - - for c in left: - if c == '*': - result |= STENO_KEYS['*'] - elif c in LEFT_KEYS: - result |= STENO_KEYS[LEFT_KEYS[c]] - elif c in VOWEL_KEYS: - result |= STENO_KEYS[VOWEL_KEYS[c]] - - for c in right: - if c in RIGHT_KEYS: - result |= STENO_KEYS[RIGHT_KEYS[c]] - + phase = 'right' + result |= STENO_KEYS['-' + c] + else: + result |= STENO_KEYS['-' + c] + else: + for c in s: + if c in LEFT_BANK: + result |= STENO_KEYS[c + '-'] return result -def parse_outline(outline: str) -> List[int]: - """Parse a multi-stroke outline like 'TEFT/-G' into stroke masks.""" - return [parse_stroke(s) for s in outline.split('/')] +def parse_outline(outline: str) -> Tuple[int, ...]: + return tuple(parse_stroke(s) for s in outline.split('/')) -def popcount(x: int) -> int: - return bin(x).count('1') +# --- CRC32 (matching Javelin's implementation) --- + +def crc32_hash(data: bytes) -> int: + return zlib.crc32(data) & 0xFFFFFFFF -def build_compact_map_dict(name: str, entries: Dict[Tuple[int, ...], str]) -> bytes: - """Build a compact map dictionary binary for a single stroke length group. +def stroke_hash(strokes: Tuple[int, ...]) -> int: + data = b''.join(struct.pack(' bytes: + return struct.pack(' int: + return (v + align - 1) & ~(align - 1) + + +def popcount32(v: int) -> int: + return bin(v & 0xFFFFFFFF).count('1') + + +class CompactMapBuilder: + """Build a compact map dictionary for a set of entries with the same stroke count.""" + + def __init__(self, stroke_length: int, entries: Dict[Tuple[int, ...], int]): + """entries maps stroke tuples to text block offsets.""" + self.stroke_length = stroke_length + self.entries = entries + + def build(self) -> Tuple[bytes, bytes, int]: + """Returns (data_block, hash_blocks, hashMapMask).""" + if not self.entries: + return b'', b'', 0 + + # Determine hash map size: next power of 2 * 128, at least 2x entries + n = len(self.entries) + num_blocks = max(1, (n * 2 + 127) // 128) + # Round up to power of 2 + p = 1 + while p < num_blocks: + p *= 2 + num_blocks = p + total_slots = num_blocks * 128 + hash_map_mask = total_slots - 1 + + # Place entries using open addressing (linear probing) + slots = [None] * total_slots + for strokes, text_offset in self.entries.items(): + h = stroke_hash(strokes) & hash_map_mask + while slots[h] is not None: + h = (h + 1) & hash_map_mask + slots[h] = (strokes, text_offset) + + # Build data block (entries in slot order) and hash map blocks. + # Data entry = textOffset(3 bytes) + strokes(3 bytes each) + entry_size = 3 + 3 * self.stroke_length + data_block = bytearray() + hash_blocks = bytearray() + + for block_idx in range(num_blocks): + masks = [0, 0, 0, 0] + block_entries = [] + + for bit in range(128): + slot_idx = block_idx * 128 + bit + if slots[slot_idx] is not None: + masks[bit // 32] |= 1 << (bit % 32) + block_entries.append(slots[slot_idx]) + + # baseOffset: count of all entries in previous blocks, + # MINUS popcount of current block's masks (Javelin convention). + # From compact_map_dictionary.cc GetOffset(): + # result = PopCount(mask << (31 - bitIndex)) + baseOffset + # + PopCount of masks[0..maskIndex-1] + # The builder sets baseOffset = running_total - PopCount(all masks in block) + running_count = len(data_block) // entry_size + block_popcount = sum(popcount32(m) for m in masks) + base_offset = running_count - block_popcount + + for m in masks: + hash_blocks.extend(struct.pack(' bytes: + # Parse all entries, group by stroke count per dictionary + parsed_dicts = [] + for dict_name, raw_dict in dict_entries: + by_length = defaultdict(dict) + max_outline_len = 0 + for outline, translation in raw_dict.items(): + try: + strokes = parse_outline(outline) + except (KeyError, ValueError): + continue + stroke_count = len(strokes) + by_length[stroke_count][strokes] = translation + max_outline_len = max(max_outline_len, stroke_count) + parsed_dicts.append((dict_name, by_length, max_outline_len)) - # Simple hash function matching Javelin's - def entry_hash(strokes): - h = 0 - for s in strokes: - h = ((h * 0x100000001B3) ^ s) & 0xFFFFFFFFFFFFFFFF - return h & 0xFFFFFFFF + # Build shared text block (deduplicated) + all_texts = set() + for _, by_length, _ in parsed_dicts: + for entries in by_length.values(): + all_texts.update(entries.values()) - # Place entries in hash map - slots = [None] * total_slots - for strokes, text in entry_list: - h = entry_hash(strokes) % total_slots - while slots[h] is not None: - h = (h + 1) % total_slots - slots[h] = (strokes, text) + text_list = sorted(all_texts) + text_to_offset = {} + text_block = bytearray() + for text in text_list: + text_to_offset[text] = len(text_block) + encoded = text.encode('utf-8') + text_block.extend(encoded) + text_block.append(0) - # Build blocks (128 entries per block) - num_blocks = hash_map_size - block_data = bytearray() - entry_data = bytearray() - running_offset = 0 + # Replace translation strings with text offsets + for _, by_length, _ in parsed_dicts: + for stroke_len in by_length: + entries = by_length[stroke_len] + by_length[stroke_len] = { + strokes: text_to_offset[text] + for strokes, text in entries.items() + } - for block_idx in range(num_blocks): - masks = [0, 0, 0, 0] - block_entries = [] + # Build compact map data for each (dict, stroke_length) + # Each dict has a StrokesDefinition array indexed by stroke_length + dict_stroke_data = [] + for dict_name, by_length, max_outline_len in parsed_dicts: + stroke_defs = [] + for length in range(1, max_outline_len + 1): + entries = by_length.get(length, {}) + builder = CompactMapBuilder(length, entries) + data_block, hash_blocks, hash_map_mask = builder.build() + stroke_defs.append((length, data_block, hash_blocks, hash_map_mask)) + dict_stroke_data.append((dict_name, stroke_defs, max_outline_len)) - for bit in range(128): - slot_idx = block_idx * 128 + bit - if slot_idx < total_slots and slots[slot_idx] is not None: - masks[bit // 32] |= 1 << (bit % 32) - block_entries.append(slots[slot_idx]) + # Now lay out the binary. We do two passes: + # 1. Calculate sizes to determine offsets + # 2. Write the actual data with correct pointers - # Write block header: 4 masks + baseOffset - for m in masks: - block_data.extend(struct.pack(' textBlock (count + ptr = 4 + ptr_size) + # SizedList prefixes + # SizedList suffixes + # uint32_t timestamp + # XipPointer dictionaries[] - running_offset += len(block_entries) + # On ARM32 (nRF52840): pointers are 4 bytes + PTR = 4 - return text_block, block_data, entry_data, total_slots - 1 # hashMapMask + # SizedList = { size_t count; T* data; } = 4 + PTR + SIZED_LIST = 4 + PTR + header_size = ( + 4 + # magic + 2 + # dictionaryCount + 1 + # hasReverseLookup + 1 + # padding + SIZED_LIST + # textBlock + SIZED_LIST + # prefixes + SIZED_LIST + # suffixes + 4 # timestamp + ) + dict_ptrs_size = len(parsed_dicts) * PTR -def build_collection(dict_entries: List[Tuple[str, Dict[str, str]]]) -> bytes: - """Build a complete StenoDictionaryCollection binary.""" + # StenoCompactMapDictionaryDefinition: + # uint8_t defaultEnabled, maximumOutlineLength, type, options (= 4) + # XipPointer name (= PTR) + # const uint8_t* textBlock (= PTR) + # const StrokesDefinition* strokes (= PTR) + COMPACT_DEF_SIZE = 4 + PTR + PTR + PTR - MAGIC = 0x3443534A # 'JSC4' + # StenoCompactMapDictionaryStrokesDefinition: + # size_t hashMapMask (= 4) + # const uint8_t* data (= PTR) + # const Block* offsets (= PTR) + STROKES_DEF_SIZE = 4 + PTR + PTR - # Parse all entries, group by (dict_name, stroke_count) - all_parsed = [] - for dict_name, raw_dict in dict_entries: - grouped = defaultdict(dict) - for outline, translation in raw_dict.items(): - strokes = parse_outline(outline) - stroke_tuple = tuple(strokes) - stroke_count = len(strokes) - grouped[stroke_count][stroke_tuple] = translation - all_parsed.append((dict_name, grouped)) + # Layout plan + offset = header_size + dict_ptrs_size - # For now, build a minimal collection with a single flat text block - # and compact map dictionaries. - # - # The full JSC4 format is complex (pointer-based, XIP-aware). - # This simplified version produces a valid binary that Javelin can load. + # Align + offset = round_up(offset, 4) - # Collect all text into one block - all_text = set() - for dict_name, grouped in all_parsed: - for stroke_count, entries in grouped.items(): - all_text.update(entries.values()) + # Dictionary definitions + dict_def_offsets = [] + for i in range(len(parsed_dicts)): + dict_def_offsets.append(offset) + offset += COMPACT_DEF_SIZE - text_list = sorted(all_text) - text_to_offset = {} - text_block = bytearray() - for text in text_list: - text_to_offset[text] = len(text_block) - text_block.extend(text.encode('utf-8')) - text_block.append(0) + offset = round_up(offset, 4) - # This is a simplified placeholder format. - # A full implementation would need to match Javelin's exact binary layout - # with XipPointer indirection, StenoDictionaryDefinition headers, etc. - # - # For production use, the recommended path is to use the Javelin web tool - # at lim.au to generate the binary, then place it at dicts/steno_dict.bin. + # Dictionary names (null-terminated strings) + dict_name_offsets = [] + for dict_name, _, _ in dict_stroke_data: + dict_name_offsets.append(offset) + offset += len(dict_name.encode('utf-8')) + 1 - print(f"WARNING: This compiler produces a simplified format.", file=sys.stderr) - print(f"For production use, generate your dictionary binary using", file=sys.stderr) - print(f"the Javelin firmware builder at https://lim.au", file=sys.stderr) - print(f"", file=sys.stderr) - print(f"Parsed {sum(len(d) for _, d in dict_entries)} entries", file=sys.stderr) - print(f"Text block: {len(text_block)} bytes", file=sys.stderr) + offset = round_up(offset, 4) - # TODO: Implement full JSC4 binary format. - # The format requires exact memory layout matching because Javelin - # casts raw flash pointers to C++ struct types (zero-copy XIP). - # This means the binary must have: - # - StenoDictionaryCollection header at offset 0 - # - StenoDictionaryDefinition pointers (XIP addresses) - # - StenoCompactMapDictionaryDefinition structs - # - Hash map blocks (StenoCompactHashMapEntryBlock) - # - Entry data (stroke + text offset pairs) - # - Text block (null-terminated strings) - # - Timestamp at end of text block (4 bytes, matches header) - # - # All pointers must be absolute flash addresses (XIP), not offsets. - # The exact base address depends on the flash partition layout. + # Strokes definitions arrays + strokes_def_offsets = [] # [(dict_idx, [(length, offset)])] + for di, (_, stroke_defs, max_len) in enumerate(dict_stroke_data): + arr_offset = offset + arr = [] + for si in range(len(stroke_defs)): + arr.append(offset) + offset += STROKES_DEF_SIZE + strokes_def_offsets.append((arr_offset, arr)) - return None + offset = round_up(offset, 4) + + # Data blocks (per dict, per stroke length) + data_offsets = [] # [[(length, offset)]] + for di, (_, stroke_defs, _) in enumerate(dict_stroke_data): + doffs = [] + for si, (length, data_block, hash_blocks, _) in enumerate(stroke_defs): + doffs.append(offset) + offset += len(data_block) + data_offsets.append(doffs) + + offset = round_up(offset, 4) + + # Hash map blocks (per dict, per stroke length) + hash_offsets = [] + for di, (_, stroke_defs, _) in enumerate(dict_stroke_data): + hoffs = [] + for si, (length, data_block, hash_blocks, _) in enumerate(stroke_defs): + hoffs.append(offset) + offset += len(hash_blocks) + hash_offsets.append(hoffs) + + offset = round_up(offset, 4) + + # Text block + text_block_offset = offset + offset += len(text_block) + + # Timestamp at end of text block + timestamp_end_offset = offset + offset += 4 + + total_size = offset + + # --- Pass 2: write binary --- + + buf = bytearray(total_size) + base = self.base_addr + + def write_u8(off, v): + buf[off] = v & 0xFF + + def write_u16(off, v): + struct.pack_into(' 0: + items = list(raw.items())[:args.max_entries] + raw = dict(items) + name = args.name[i] if args.name and i < len(args.name) else input_path.rsplit('/', 1)[-1].rsplit('.', 1)[0] dict_entries.append((name, raw)) - print(f"Loaded {input_path}: {len(raw)} entries", file=sys.stderr) + print(f"Loaded {input_path}: {len(raw)} entries as '{name}'", file=sys.stderr) - result = build_collection(dict_entries) - - if result is None: - print("", file=sys.stderr) - print("Full binary compilation not yet implemented.", file=sys.stderr) - print("To get a working dictionary binary:", file=sys.stderr) - print(" 1. Go to https://lim.au", file=sys.stderr) - print(" 2. Select your theory (Plover or Lapwing)", file=sys.stderr) - print(" 3. Download the firmware", file=sys.stderr) - print(" 4. Extract the dictionary binary from the firmware", file=sys.stderr) - print(" OR", file=sys.stderr) - print(" Use the Javelin console commands to upload dictionaries", file=sys.stderr) - print(" over USB at runtime.", file=sys.stderr) - sys.exit(1) + builder = DictionaryCollectionBuilder(base_addr=args.base_addr) + result = builder.build(dict_entries) with open(args.output, 'wb') as f: f.write(result) - print(f"Wrote {len(result)} bytes to {args.output}", file=sys.stderr) + + print(f"Wrote {len(result)} bytes ({len(result)/1024:.1f} KB) to {args.output}", file=sys.stderr) if __name__ == '__main__': diff --git a/tools/dict_compiler/download_dicts.sh b/tools/dict_compiler/download_dicts.sh new file mode 100755 index 0000000..b69680a --- /dev/null +++ b/tools/dict_compiler/download_dicts.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Download Plover and Lapwing steno dictionaries from their official repos. +# Usage: ./download_dicts.sh [plover|lapwing|all] [output_dir] + +set -euo pipefail + +THEORY="${1:-all}" +OUTDIR="${2:-./dicts_src}" + +mkdir -p "$OUTDIR" + +download_plover() { + echo "Downloading Plover main.json..." + curl -fsSL \ + "https://raw.githubusercontent.com/openstenoproject/plover/main/plover/assets/main.json" \ + -o "$OUTDIR/plover-main.json" + echo " $(python3 -c "import json; print(len(json.load(open('$OUTDIR/plover-main.json'))))" 2>/dev/null || echo '?') entries" +} + +download_lapwing() { + local BASE="https://raw.githubusercontent.com/aerickt/plover-lapwing-aio/main/plover_lapwing/dictionaries" + echo "Downloading Lapwing dictionaries..." + for f in lapwing-base lapwing-commands lapwing-numbers; do + curl -fsSL "$BASE/$f.json" -o "$OUTDIR/$f.json" + echo " $f.json: $(python3 -c "import json; print(len(json.load(open('$OUTDIR/$f.json'))))" 2>/dev/null || echo '?') entries" + done +} + +case "$THEORY" in + plover) download_plover ;; + lapwing) download_lapwing ;; + all) download_plover; download_lapwing ;; + *) echo "Usage: $0 [plover|lapwing|all] [output_dir]"; exit 1 ;; +esac + +echo "Done. Dictionaries saved to $OUTDIR/"