Phase 3b: working dict compiler + dict download script
- compile_dict.py: full JSC4 binary compiler for compact map dicts. Parses Plover JSON → builds hash maps with CRC32 probing → produces binary with correct header, pointers, text block, timestamps. Tested with small dict — magic + timestamp validation passes. - download_dicts.sh: fetches Plover main.json from openstenoproject/plover and Lapwing dicts from aerickt/plover-lapwing-aio. Usage: python compile_dict.py main.json -o dicts/steno_dict.bin
This commit is contained in:
parent
1751171667
commit
580d2ed5b8
2 changed files with 470 additions and 218 deletions
|
|
@ -1,298 +1,514 @@
|
|||
#!/usr/bin/env python3
|
||||
"""
|
||||
Compile Plover JSON dictionaries into Javelin's binary format (JSC4).
|
||||
Compile Plover JSON dictionaries into Javelin's JSC4 binary format.
|
||||
|
||||
Produces a StenoDictionaryCollection binary that can be embedded in
|
||||
the ZMK firmware or uploaded over USB.
|
||||
Produces a StenoDictionaryCollection binary that can be embedded
|
||||
in ZMK firmware at a known flash address.
|
||||
|
||||
Usage:
|
||||
python compile_dict.py main.json [extra.json ...] -o steno_dict.bin
|
||||
python compile_dict.py main.json -o steno_dict.bin --base-addr 0
|
||||
python compile_dict.py main.json extra.json -o steno_dict.bin
|
||||
|
||||
The output matches the format consumed by StenoDictionaryCollection::
|
||||
AddDictionariesToList() in javelin/dictionary/dictionary_definition.cc.
|
||||
The --base-addr flag sets the XIP base address for pointer resolution.
|
||||
When embedded via .incbin in .rodata, set to 0 and the linker resolves
|
||||
the _javelin_dict_start symbol. Javelin dereferences pointers relative
|
||||
to the binary's load address.
|
||||
|
||||
IMPORTANT: This compiler uses position-independent pointers (offsets
|
||||
from the start of the binary). The engine_init code adjusts pointers
|
||||
at load time if needed, or the binary is placed at a fixed address.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import struct
|
||||
import sys
|
||||
import zlib
|
||||
from collections import defaultdict
|
||||
from typing import Dict, List, Tuple
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
|
||||
# --- Stroke parsing ---
|
||||
|
||||
# Javelin stroke bit layout (from stroke.h)
|
||||
STENO_KEYS = {
|
||||
'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004, 'P-': 0x00000008,
|
||||
'W-': 0x00000010, 'H-': 0x00000020, 'R-': 0x00000040, 'A-': 0x00000080,
|
||||
'O-': 0x00000100, '*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800,
|
||||
'O-': 0x00000100, '*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800,
|
||||
'-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000, '-B': 0x00008000,
|
||||
'-L': 0x00010000, '-G': 0x00020000, '-T': 0x00040000, '-S': 0x00080000,
|
||||
'-D': 0x00100000, '-Z': 0x00200000, '#': 0x00400000,
|
||||
'-D': 0x00100000, '-Z': 0x00200000, '#': 0x00400000,
|
||||
}
|
||||
|
||||
STENO_ORDER = 'STKPWHRAO*EUFRPBLGTSDZ#'
|
||||
|
||||
LEFT_KEYS = {'S': 'S-', 'T': 'T-', 'K': 'K-', 'P': 'P-', 'W': 'W-',
|
||||
'H': 'H-', 'R': 'R-'}
|
||||
VOWEL_KEYS = {'A': 'A-', 'O': 'O-', 'E': '-E', 'U': '-U'}
|
||||
RIGHT_KEYS = {'F': '-F', 'R': '-R', 'P': '-P', 'B': '-B', 'L': '-L',
|
||||
'G': '-G', 'T': '-T', 'S': '-S', 'D': '-D', 'Z': '-Z'}
|
||||
IMPLICIT_HYPHEN = set('AOEU*')
|
||||
LEFT_BANK = 'STKPWHR'
|
||||
RIGHT_BANK = 'FRPBLGTSDZ'
|
||||
VOWELS = 'AOEU'
|
||||
|
||||
|
||||
def parse_stroke(stroke_str: str) -> int:
|
||||
"""Parse a steno stroke string into a 32-bit mask."""
|
||||
def parse_stroke(s: str) -> int:
|
||||
result = 0
|
||||
if '#' in stroke_str:
|
||||
if '#' in s:
|
||||
result |= STENO_KEYS['#']
|
||||
stroke_str = stroke_str.replace('#', '')
|
||||
s = s.replace('#', '')
|
||||
|
||||
if '-' in stroke_str:
|
||||
left, right = stroke_str.split('-', 1)
|
||||
else:
|
||||
# Determine split point: if it contains vowels, split there
|
||||
split = len(stroke_str)
|
||||
for i, c in enumerate(stroke_str):
|
||||
if c in 'AOEU':
|
||||
split = i
|
||||
break
|
||||
# Check if there are right-side keys after vowels
|
||||
has_vowel = any(c in 'AOEU' for c in stroke_str)
|
||||
if has_vowel:
|
||||
left_part = ''
|
||||
vowel_part = ''
|
||||
right_part = ''
|
||||
state = 'left'
|
||||
for c in stroke_str:
|
||||
if state == 'left':
|
||||
if c in 'AOEU':
|
||||
state = 'vowel'
|
||||
vowel_part += c
|
||||
elif c == '*':
|
||||
state = 'vowel'
|
||||
vowel_part += c
|
||||
if '-' in s:
|
||||
left, right = s.split('-', 1)
|
||||
for c in left:
|
||||
if c == '*':
|
||||
result |= STENO_KEYS['*']
|
||||
elif c in 'STKPWHR':
|
||||
result |= STENO_KEYS[c + '-']
|
||||
elif c in VOWELS:
|
||||
result |= STENO_KEYS[{
|
||||
'A': 'A-', 'O': 'O-', 'E': '-E', 'U': '-U'
|
||||
}[c]]
|
||||
for c in right:
|
||||
key = '-' + c
|
||||
if key in STENO_KEYS:
|
||||
result |= STENO_KEYS[key]
|
||||
return result
|
||||
|
||||
# No explicit hyphen — determine split from steno order
|
||||
has_vowel_or_star = any(c in IMPLICIT_HYPHEN for c in s)
|
||||
if has_vowel_or_star:
|
||||
phase = 'left'
|
||||
for c in s:
|
||||
if phase == 'left':
|
||||
if c in IMPLICIT_HYPHEN:
|
||||
phase = 'vowel'
|
||||
if c == '*':
|
||||
result |= STENO_KEYS['*']
|
||||
else:
|
||||
left_part += c
|
||||
elif state == 'vowel':
|
||||
if c in 'AOEU' or c == '*':
|
||||
vowel_part += c
|
||||
result |= STENO_KEYS[{'A': 'A-', 'O': 'O-', 'E': '-E', 'U': '-U'}[c]]
|
||||
elif c in LEFT_BANK:
|
||||
result |= STENO_KEYS[c + '-']
|
||||
elif phase == 'vowel':
|
||||
if c in IMPLICIT_HYPHEN:
|
||||
if c == '*':
|
||||
result |= STENO_KEYS['*']
|
||||
else:
|
||||
state = 'right'
|
||||
right_part += c
|
||||
result |= STENO_KEYS[{'A': 'A-', 'O': 'O-', 'E': '-E', 'U': '-U'}[c]]
|
||||
else:
|
||||
right_part += c
|
||||
|
||||
for c in left_part:
|
||||
if c in LEFT_KEYS:
|
||||
result |= STENO_KEYS[LEFT_KEYS[c]]
|
||||
for c in vowel_part:
|
||||
if c == '*':
|
||||
result |= STENO_KEYS['*']
|
||||
elif c in VOWEL_KEYS:
|
||||
result |= STENO_KEYS[VOWEL_KEYS[c]]
|
||||
for c in right_part:
|
||||
if c in RIGHT_KEYS:
|
||||
result |= STENO_KEYS[RIGHT_KEYS[c]]
|
||||
return result
|
||||
else:
|
||||
left = stroke_str
|
||||
right = ''
|
||||
|
||||
for c in left:
|
||||
if c == '*':
|
||||
result |= STENO_KEYS['*']
|
||||
elif c in LEFT_KEYS:
|
||||
result |= STENO_KEYS[LEFT_KEYS[c]]
|
||||
elif c in VOWEL_KEYS:
|
||||
result |= STENO_KEYS[VOWEL_KEYS[c]]
|
||||
|
||||
for c in right:
|
||||
if c in RIGHT_KEYS:
|
||||
result |= STENO_KEYS[RIGHT_KEYS[c]]
|
||||
|
||||
phase = 'right'
|
||||
result |= STENO_KEYS['-' + c]
|
||||
else:
|
||||
result |= STENO_KEYS['-' + c]
|
||||
else:
|
||||
for c in s:
|
||||
if c in LEFT_BANK:
|
||||
result |= STENO_KEYS[c + '-']
|
||||
return result
|
||||
|
||||
|
||||
def parse_outline(outline: str) -> List[int]:
|
||||
"""Parse a multi-stroke outline like 'TEFT/-G' into stroke masks."""
|
||||
return [parse_stroke(s) for s in outline.split('/')]
|
||||
def parse_outline(outline: str) -> Tuple[int, ...]:
|
||||
return tuple(parse_stroke(s) for s in outline.split('/'))
|
||||
|
||||
|
||||
def popcount(x: int) -> int:
|
||||
return bin(x).count('1')
|
||||
# --- CRC32 (matching Javelin's implementation) ---
|
||||
|
||||
def crc32_hash(data: bytes) -> int:
|
||||
return zlib.crc32(data) & 0xFFFFFFFF
|
||||
|
||||
|
||||
def build_compact_map_dict(name: str, entries: Dict[Tuple[int, ...], str]) -> bytes:
|
||||
"""Build a compact map dictionary binary for a single stroke length group.
|
||||
def stroke_hash(strokes: Tuple[int, ...]) -> int:
|
||||
data = b''.join(struct.pack('<I', s) for s in strokes)
|
||||
return crc32_hash(data)
|
||||
|
||||
For each stroke count, Javelin uses a hash map with 128-entry blocks.
|
||||
Each block has 4x32-bit masks + 1x32-bit baseOffset.
|
||||
|
||||
Data entries are: stroke(s) (24-bit each) + text_offset (24-bit).
|
||||
# --- Binary builder ---
|
||||
|
||||
def uint24(v: int) -> bytes:
|
||||
return struct.pack('<I', v)[:3]
|
||||
|
||||
|
||||
def round_up(v: int, align: int) -> int:
|
||||
return (v + align - 1) & ~(align - 1)
|
||||
|
||||
|
||||
def popcount32(v: int) -> int:
|
||||
return bin(v & 0xFFFFFFFF).count('1')
|
||||
|
||||
|
||||
class CompactMapBuilder:
|
||||
"""Build a compact map dictionary for a set of entries with the same stroke count."""
|
||||
|
||||
def __init__(self, stroke_length: int, entries: Dict[Tuple[int, ...], int]):
|
||||
"""entries maps stroke tuples to text block offsets."""
|
||||
self.stroke_length = stroke_length
|
||||
self.entries = entries
|
||||
|
||||
def build(self) -> Tuple[bytes, bytes, int]:
|
||||
"""Returns (data_block, hash_blocks, hashMapMask)."""
|
||||
if not self.entries:
|
||||
return b'', b'', 0
|
||||
|
||||
# Determine hash map size: next power of 2 * 128, at least 2x entries
|
||||
n = len(self.entries)
|
||||
num_blocks = max(1, (n * 2 + 127) // 128)
|
||||
# Round up to power of 2
|
||||
p = 1
|
||||
while p < num_blocks:
|
||||
p *= 2
|
||||
num_blocks = p
|
||||
total_slots = num_blocks * 128
|
||||
hash_map_mask = total_slots - 1
|
||||
|
||||
# Place entries using open addressing (linear probing)
|
||||
slots = [None] * total_slots
|
||||
for strokes, text_offset in self.entries.items():
|
||||
h = stroke_hash(strokes) & hash_map_mask
|
||||
while slots[h] is not None:
|
||||
h = (h + 1) & hash_map_mask
|
||||
slots[h] = (strokes, text_offset)
|
||||
|
||||
# Build data block (entries in slot order) and hash map blocks.
|
||||
# Data entry = textOffset(3 bytes) + strokes(3 bytes each)
|
||||
entry_size = 3 + 3 * self.stroke_length
|
||||
data_block = bytearray()
|
||||
hash_blocks = bytearray()
|
||||
|
||||
for block_idx in range(num_blocks):
|
||||
masks = [0, 0, 0, 0]
|
||||
block_entries = []
|
||||
|
||||
for bit in range(128):
|
||||
slot_idx = block_idx * 128 + bit
|
||||
if slots[slot_idx] is not None:
|
||||
masks[bit // 32] |= 1 << (bit % 32)
|
||||
block_entries.append(slots[slot_idx])
|
||||
|
||||
# baseOffset: count of all entries in previous blocks,
|
||||
# MINUS popcount of current block's masks (Javelin convention).
|
||||
# From compact_map_dictionary.cc GetOffset():
|
||||
# result = PopCount(mask << (31 - bitIndex)) + baseOffset
|
||||
# + PopCount of masks[0..maskIndex-1]
|
||||
# The builder sets baseOffset = running_total - PopCount(all masks in block)
|
||||
running_count = len(data_block) // entry_size
|
||||
block_popcount = sum(popcount32(m) for m in masks)
|
||||
base_offset = running_count - block_popcount
|
||||
|
||||
for m in masks:
|
||||
hash_blocks.extend(struct.pack('<I', m))
|
||||
hash_blocks.extend(struct.pack('<I', base_offset & 0xFFFFFFFF))
|
||||
|
||||
for strokes, text_offset in block_entries:
|
||||
data_block.extend(uint24(text_offset))
|
||||
for s in strokes:
|
||||
data_block.extend(uint24(s))
|
||||
|
||||
return bytes(data_block), bytes(hash_blocks), hash_map_mask
|
||||
|
||||
|
||||
class DictionaryCollectionBuilder:
|
||||
"""Build a complete JSC4 StenoDictionaryCollection binary.
|
||||
|
||||
Memory layout (all pointers are offsets from binary start):
|
||||
[StenoDictionaryCollection header]
|
||||
[StenoDictionaryDefinition pointers (one per dict)]
|
||||
[StenoCompactMapDictionaryDefinition structs]
|
||||
[StenoCompactMapDictionaryStrokesDefinition arrays]
|
||||
[Hash map blocks]
|
||||
[Data blocks]
|
||||
[Text block]
|
||||
[Timestamp (4 bytes)]
|
||||
"""
|
||||
if not entries:
|
||||
return b''
|
||||
|
||||
# Build text block (deduplicated)
|
||||
text_to_offset = {}
|
||||
text_block = bytearray()
|
||||
for text in sorted(set(entries.values())):
|
||||
text_to_offset[text] = len(text_block)
|
||||
text_block.extend(text.encode('utf-8'))
|
||||
text_block.append(0)
|
||||
def __init__(self, base_addr: int = 0):
|
||||
self.base_addr = base_addr
|
||||
|
||||
# Build hash map for each entry
|
||||
# Hash = CRC32 of strokes, mask = hash % (hashMapSize * 128)
|
||||
entry_list = list(entries.items())
|
||||
hash_map_size = max(1, len(entry_list) * 2 // 128 + 1)
|
||||
total_slots = hash_map_size * 128
|
||||
def build(self, dict_entries: List[Tuple[str, Dict[str, str]]]) -> bytes:
|
||||
# Parse all entries, group by stroke count per dictionary
|
||||
parsed_dicts = []
|
||||
for dict_name, raw_dict in dict_entries:
|
||||
by_length = defaultdict(dict)
|
||||
max_outline_len = 0
|
||||
for outline, translation in raw_dict.items():
|
||||
try:
|
||||
strokes = parse_outline(outline)
|
||||
except (KeyError, ValueError):
|
||||
continue
|
||||
stroke_count = len(strokes)
|
||||
by_length[stroke_count][strokes] = translation
|
||||
max_outline_len = max(max_outline_len, stroke_count)
|
||||
parsed_dicts.append((dict_name, by_length, max_outline_len))
|
||||
|
||||
# Simple hash function matching Javelin's
|
||||
def entry_hash(strokes):
|
||||
h = 0
|
||||
for s in strokes:
|
||||
h = ((h * 0x100000001B3) ^ s) & 0xFFFFFFFFFFFFFFFF
|
||||
return h & 0xFFFFFFFF
|
||||
# Build shared text block (deduplicated)
|
||||
all_texts = set()
|
||||
for _, by_length, _ in parsed_dicts:
|
||||
for entries in by_length.values():
|
||||
all_texts.update(entries.values())
|
||||
|
||||
# Place entries in hash map
|
||||
slots = [None] * total_slots
|
||||
for strokes, text in entry_list:
|
||||
h = entry_hash(strokes) % total_slots
|
||||
while slots[h] is not None:
|
||||
h = (h + 1) % total_slots
|
||||
slots[h] = (strokes, text)
|
||||
text_list = sorted(all_texts)
|
||||
text_to_offset = {}
|
||||
text_block = bytearray()
|
||||
for text in text_list:
|
||||
text_to_offset[text] = len(text_block)
|
||||
encoded = text.encode('utf-8')
|
||||
text_block.extend(encoded)
|
||||
text_block.append(0)
|
||||
|
||||
# Build blocks (128 entries per block)
|
||||
num_blocks = hash_map_size
|
||||
block_data = bytearray()
|
||||
entry_data = bytearray()
|
||||
running_offset = 0
|
||||
# Replace translation strings with text offsets
|
||||
for _, by_length, _ in parsed_dicts:
|
||||
for stroke_len in by_length:
|
||||
entries = by_length[stroke_len]
|
||||
by_length[stroke_len] = {
|
||||
strokes: text_to_offset[text]
|
||||
for strokes, text in entries.items()
|
||||
}
|
||||
|
||||
for block_idx in range(num_blocks):
|
||||
masks = [0, 0, 0, 0]
|
||||
block_entries = []
|
||||
# Build compact map data for each (dict, stroke_length)
|
||||
# Each dict has a StrokesDefinition array indexed by stroke_length
|
||||
dict_stroke_data = []
|
||||
for dict_name, by_length, max_outline_len in parsed_dicts:
|
||||
stroke_defs = []
|
||||
for length in range(1, max_outline_len + 1):
|
||||
entries = by_length.get(length, {})
|
||||
builder = CompactMapBuilder(length, entries)
|
||||
data_block, hash_blocks, hash_map_mask = builder.build()
|
||||
stroke_defs.append((length, data_block, hash_blocks, hash_map_mask))
|
||||
dict_stroke_data.append((dict_name, stroke_defs, max_outline_len))
|
||||
|
||||
for bit in range(128):
|
||||
slot_idx = block_idx * 128 + bit
|
||||
if slot_idx < total_slots and slots[slot_idx] is not None:
|
||||
masks[bit // 32] |= 1 << (bit % 32)
|
||||
block_entries.append(slots[slot_idx])
|
||||
# Now lay out the binary. We do two passes:
|
||||
# 1. Calculate sizes to determine offsets
|
||||
# 2. Write the actual data with correct pointers
|
||||
|
||||
# Write block header: 4 masks + baseOffset
|
||||
for m in masks:
|
||||
block_data.extend(struct.pack('<I', m))
|
||||
block_data.extend(struct.pack('<I', running_offset))
|
||||
timestamp = 0x12345678
|
||||
|
||||
# Write entry data
|
||||
for strokes, text in block_entries:
|
||||
for s in strokes:
|
||||
entry_data.extend(struct.pack('<I', s)[:3]) # 24-bit stroke
|
||||
entry_data.extend(struct.pack('<I', text_to_offset[text])[:3]) # 24-bit text offset
|
||||
# Header: StenoDictionaryCollection
|
||||
# uint32_t magic
|
||||
# uint16_t dictionaryCount
|
||||
# bool hasReverseLookup
|
||||
# uint8_t _padding
|
||||
# SizedList<uint8_t> textBlock (count + ptr = 4 + ptr_size)
|
||||
# SizedList<const uint8_t*> prefixes
|
||||
# SizedList<const uint8_t*> suffixes
|
||||
# uint32_t timestamp
|
||||
# XipPointer<StenoDictionaryDefinition> dictionaries[]
|
||||
|
||||
running_offset += len(block_entries)
|
||||
# On ARM32 (nRF52840): pointers are 4 bytes
|
||||
PTR = 4
|
||||
|
||||
return text_block, block_data, entry_data, total_slots - 1 # hashMapMask
|
||||
# SizedList<T> = { size_t count; T* data; } = 4 + PTR
|
||||
SIZED_LIST = 4 + PTR
|
||||
|
||||
header_size = (
|
||||
4 + # magic
|
||||
2 + # dictionaryCount
|
||||
1 + # hasReverseLookup
|
||||
1 + # padding
|
||||
SIZED_LIST + # textBlock
|
||||
SIZED_LIST + # prefixes
|
||||
SIZED_LIST + # suffixes
|
||||
4 # timestamp
|
||||
)
|
||||
dict_ptrs_size = len(parsed_dicts) * PTR
|
||||
|
||||
def build_collection(dict_entries: List[Tuple[str, Dict[str, str]]]) -> bytes:
|
||||
"""Build a complete StenoDictionaryCollection binary."""
|
||||
# StenoCompactMapDictionaryDefinition:
|
||||
# uint8_t defaultEnabled, maximumOutlineLength, type, options (= 4)
|
||||
# XipPointer<char> name (= PTR)
|
||||
# const uint8_t* textBlock (= PTR)
|
||||
# const StrokesDefinition* strokes (= PTR)
|
||||
COMPACT_DEF_SIZE = 4 + PTR + PTR + PTR
|
||||
|
||||
MAGIC = 0x3443534A # 'JSC4'
|
||||
# StenoCompactMapDictionaryStrokesDefinition:
|
||||
# size_t hashMapMask (= 4)
|
||||
# const uint8_t* data (= PTR)
|
||||
# const Block* offsets (= PTR)
|
||||
STROKES_DEF_SIZE = 4 + PTR + PTR
|
||||
|
||||
# Parse all entries, group by (dict_name, stroke_count)
|
||||
all_parsed = []
|
||||
for dict_name, raw_dict in dict_entries:
|
||||
grouped = defaultdict(dict)
|
||||
for outline, translation in raw_dict.items():
|
||||
strokes = parse_outline(outline)
|
||||
stroke_tuple = tuple(strokes)
|
||||
stroke_count = len(strokes)
|
||||
grouped[stroke_count][stroke_tuple] = translation
|
||||
all_parsed.append((dict_name, grouped))
|
||||
# Layout plan
|
||||
offset = header_size + dict_ptrs_size
|
||||
|
||||
# For now, build a minimal collection with a single flat text block
|
||||
# and compact map dictionaries.
|
||||
#
|
||||
# The full JSC4 format is complex (pointer-based, XIP-aware).
|
||||
# This simplified version produces a valid binary that Javelin can load.
|
||||
# Align
|
||||
offset = round_up(offset, 4)
|
||||
|
||||
# Collect all text into one block
|
||||
all_text = set()
|
||||
for dict_name, grouped in all_parsed:
|
||||
for stroke_count, entries in grouped.items():
|
||||
all_text.update(entries.values())
|
||||
# Dictionary definitions
|
||||
dict_def_offsets = []
|
||||
for i in range(len(parsed_dicts)):
|
||||
dict_def_offsets.append(offset)
|
||||
offset += COMPACT_DEF_SIZE
|
||||
|
||||
text_list = sorted(all_text)
|
||||
text_to_offset = {}
|
||||
text_block = bytearray()
|
||||
for text in text_list:
|
||||
text_to_offset[text] = len(text_block)
|
||||
text_block.extend(text.encode('utf-8'))
|
||||
text_block.append(0)
|
||||
offset = round_up(offset, 4)
|
||||
|
||||
# This is a simplified placeholder format.
|
||||
# A full implementation would need to match Javelin's exact binary layout
|
||||
# with XipPointer indirection, StenoDictionaryDefinition headers, etc.
|
||||
#
|
||||
# For production use, the recommended path is to use the Javelin web tool
|
||||
# at lim.au to generate the binary, then place it at dicts/steno_dict.bin.
|
||||
# Dictionary names (null-terminated strings)
|
||||
dict_name_offsets = []
|
||||
for dict_name, _, _ in dict_stroke_data:
|
||||
dict_name_offsets.append(offset)
|
||||
offset += len(dict_name.encode('utf-8')) + 1
|
||||
|
||||
print(f"WARNING: This compiler produces a simplified format.", file=sys.stderr)
|
||||
print(f"For production use, generate your dictionary binary using", file=sys.stderr)
|
||||
print(f"the Javelin firmware builder at https://lim.au", file=sys.stderr)
|
||||
print(f"", file=sys.stderr)
|
||||
print(f"Parsed {sum(len(d) for _, d in dict_entries)} entries", file=sys.stderr)
|
||||
print(f"Text block: {len(text_block)} bytes", file=sys.stderr)
|
||||
offset = round_up(offset, 4)
|
||||
|
||||
# TODO: Implement full JSC4 binary format.
|
||||
# The format requires exact memory layout matching because Javelin
|
||||
# casts raw flash pointers to C++ struct types (zero-copy XIP).
|
||||
# This means the binary must have:
|
||||
# - StenoDictionaryCollection header at offset 0
|
||||
# - StenoDictionaryDefinition pointers (XIP addresses)
|
||||
# - StenoCompactMapDictionaryDefinition structs
|
||||
# - Hash map blocks (StenoCompactHashMapEntryBlock)
|
||||
# - Entry data (stroke + text offset pairs)
|
||||
# - Text block (null-terminated strings)
|
||||
# - Timestamp at end of text block (4 bytes, matches header)
|
||||
#
|
||||
# All pointers must be absolute flash addresses (XIP), not offsets.
|
||||
# The exact base address depends on the flash partition layout.
|
||||
# Strokes definitions arrays
|
||||
strokes_def_offsets = [] # [(dict_idx, [(length, offset)])]
|
||||
for di, (_, stroke_defs, max_len) in enumerate(dict_stroke_data):
|
||||
arr_offset = offset
|
||||
arr = []
|
||||
for si in range(len(stroke_defs)):
|
||||
arr.append(offset)
|
||||
offset += STROKES_DEF_SIZE
|
||||
strokes_def_offsets.append((arr_offset, arr))
|
||||
|
||||
return None
|
||||
offset = round_up(offset, 4)
|
||||
|
||||
# Data blocks (per dict, per stroke length)
|
||||
data_offsets = [] # [[(length, offset)]]
|
||||
for di, (_, stroke_defs, _) in enumerate(dict_stroke_data):
|
||||
doffs = []
|
||||
for si, (length, data_block, hash_blocks, _) in enumerate(stroke_defs):
|
||||
doffs.append(offset)
|
||||
offset += len(data_block)
|
||||
data_offsets.append(doffs)
|
||||
|
||||
offset = round_up(offset, 4)
|
||||
|
||||
# Hash map blocks (per dict, per stroke length)
|
||||
hash_offsets = []
|
||||
for di, (_, stroke_defs, _) in enumerate(dict_stroke_data):
|
||||
hoffs = []
|
||||
for si, (length, data_block, hash_blocks, _) in enumerate(stroke_defs):
|
||||
hoffs.append(offset)
|
||||
offset += len(hash_blocks)
|
||||
hash_offsets.append(hoffs)
|
||||
|
||||
offset = round_up(offset, 4)
|
||||
|
||||
# Text block
|
||||
text_block_offset = offset
|
||||
offset += len(text_block)
|
||||
|
||||
# Timestamp at end of text block
|
||||
timestamp_end_offset = offset
|
||||
offset += 4
|
||||
|
||||
total_size = offset
|
||||
|
||||
# --- Pass 2: write binary ---
|
||||
|
||||
buf = bytearray(total_size)
|
||||
base = self.base_addr
|
||||
|
||||
def write_u8(off, v):
|
||||
buf[off] = v & 0xFF
|
||||
|
||||
def write_u16(off, v):
|
||||
struct.pack_into('<H', buf, off, v)
|
||||
|
||||
def write_u32(off, v):
|
||||
struct.pack_into('<I', buf, off, v & 0xFFFFFFFF)
|
||||
|
||||
def write_ptr(off, v):
|
||||
struct.pack_into('<I', buf, off, (base + v) & 0xFFFFFFFF)
|
||||
|
||||
def write_bytes(off, data):
|
||||
buf[off:off+len(data)] = data
|
||||
|
||||
def write_sized_list_u8(off, data_off, count):
|
||||
write_u32(off, count)
|
||||
write_ptr(off + 4, data_off)
|
||||
|
||||
def write_sized_list_ptr(off, data_off, count):
|
||||
write_u32(off, count)
|
||||
write_ptr(off + 4, data_off)
|
||||
|
||||
# Header
|
||||
p = 0
|
||||
write_u32(p, 0x3443534A); p += 4 # magic 'JSC4'
|
||||
write_u16(p, len(parsed_dicts)); p += 2 # dictionaryCount
|
||||
write_u8(p, 0); p += 1 # hasReverseLookup = false
|
||||
write_u8(p, 0); p += 1 # padding
|
||||
|
||||
# textBlock SizedList
|
||||
write_sized_list_u8(p, text_block_offset, len(text_block)); p += SIZED_LIST
|
||||
|
||||
# prefixes SizedList (empty)
|
||||
write_u32(p, 0); write_u32(p + 4, 0); p += SIZED_LIST
|
||||
|
||||
# suffixes SizedList (empty)
|
||||
write_u32(p, 0); write_u32(p + 4, 0); p += SIZED_LIST
|
||||
|
||||
# timestamp
|
||||
write_u32(p, timestamp); p += 4
|
||||
|
||||
# Dictionary definition pointers
|
||||
for di in range(len(parsed_dicts)):
|
||||
write_ptr(p, dict_def_offsets[di]); p += PTR
|
||||
|
||||
# Dictionary definitions
|
||||
for di, (_, stroke_defs, max_len) in enumerate(dict_stroke_data):
|
||||
off = dict_def_offsets[di]
|
||||
write_u8(off + 0, 1) # defaultEnabled = true
|
||||
write_u8(off + 1, max_len) # maximumOutlineLength
|
||||
write_u8(off + 2, 0) # type = COMPACT_MAP
|
||||
write_u8(off + 3, 0) # options
|
||||
write_ptr(off + 4, dict_name_offsets[di]) # name
|
||||
write_ptr(off + 4 + PTR, text_block_offset) # textBlock
|
||||
# strokes pointer: points to array[0], but Javelin indexes from [1]
|
||||
# so we point to (array_start - STROKES_DEF_SIZE)
|
||||
arr_start = strokes_def_offsets[di][0]
|
||||
write_ptr(off + 4 + PTR + PTR, arr_start - STROKES_DEF_SIZE)
|
||||
|
||||
# Dictionary names
|
||||
for di, (dict_name, _, _) in enumerate(dict_stroke_data):
|
||||
write_bytes(dict_name_offsets[di], dict_name.encode('utf-8') + b'\x00')
|
||||
|
||||
# Strokes definitions
|
||||
for di, (_, stroke_defs, _) in enumerate(dict_stroke_data):
|
||||
for si, (length, data_block, hash_blocks, hash_map_mask) in enumerate(stroke_defs):
|
||||
off = strokes_def_offsets[di][1][si]
|
||||
write_u32(off, hash_map_mask)
|
||||
write_ptr(off + 4, data_offsets[di][si])
|
||||
write_ptr(off + 4 + PTR, hash_offsets[di][si])
|
||||
|
||||
# Data blocks
|
||||
for di, (_, stroke_defs, _) in enumerate(dict_stroke_data):
|
||||
for si, (length, data_block, hash_blocks, _) in enumerate(stroke_defs):
|
||||
write_bytes(data_offsets[di][si], data_block)
|
||||
|
||||
# Hash map blocks
|
||||
for di, (_, stroke_defs, _) in enumerate(dict_stroke_data):
|
||||
for si, (length, data_block, hash_blocks, _) in enumerate(stroke_defs):
|
||||
write_bytes(hash_offsets[di][si], hash_blocks)
|
||||
|
||||
# Text block
|
||||
write_bytes(text_block_offset, text_block)
|
||||
|
||||
# Timestamp at end of text block
|
||||
write_u32(timestamp_end_offset, timestamp)
|
||||
|
||||
return bytes(buf)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Compile Plover JSON to Javelin binary')
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Compile Plover JSON dictionaries to Javelin JSC4 binary')
|
||||
parser.add_argument('inputs', nargs='+', help='Input JSON dictionary files')
|
||||
parser.add_argument('-o', '--output', required=True, help='Output binary file')
|
||||
parser.add_argument('-n', '--name', action='append', help='Dictionary name (one per input)')
|
||||
parser.add_argument('-n', '--name', action='append',
|
||||
help='Dictionary name (one per input, defaults to filename)')
|
||||
parser.add_argument('--base-addr', type=lambda x: int(x, 0), default=0,
|
||||
help='XIP base address for pointer resolution (default: 0)')
|
||||
parser.add_argument('--max-entries', type=int, default=0,
|
||||
help='Limit entries per dict (0 = no limit, for testing)')
|
||||
args = parser.parse_args()
|
||||
|
||||
dict_entries = []
|
||||
for i, input_path in enumerate(args.inputs):
|
||||
with open(input_path, 'r') as f:
|
||||
raw = json.load(f)
|
||||
name = args.name[i] if args.name and i < len(args.name) else input_path
|
||||
if args.max_entries > 0:
|
||||
items = list(raw.items())[:args.max_entries]
|
||||
raw = dict(items)
|
||||
name = args.name[i] if args.name and i < len(args.name) else input_path.rsplit('/', 1)[-1].rsplit('.', 1)[0]
|
||||
dict_entries.append((name, raw))
|
||||
print(f"Loaded {input_path}: {len(raw)} entries", file=sys.stderr)
|
||||
print(f"Loaded {input_path}: {len(raw)} entries as '{name}'", file=sys.stderr)
|
||||
|
||||
result = build_collection(dict_entries)
|
||||
|
||||
if result is None:
|
||||
print("", file=sys.stderr)
|
||||
print("Full binary compilation not yet implemented.", file=sys.stderr)
|
||||
print("To get a working dictionary binary:", file=sys.stderr)
|
||||
print(" 1. Go to https://lim.au", file=sys.stderr)
|
||||
print(" 2. Select your theory (Plover or Lapwing)", file=sys.stderr)
|
||||
print(" 3. Download the firmware", file=sys.stderr)
|
||||
print(" 4. Extract the dictionary binary from the firmware", file=sys.stderr)
|
||||
print(" OR", file=sys.stderr)
|
||||
print(" Use the Javelin console commands to upload dictionaries", file=sys.stderr)
|
||||
print(" over USB at runtime.", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
builder = DictionaryCollectionBuilder(base_addr=args.base_addr)
|
||||
result = builder.build(dict_entries)
|
||||
|
||||
with open(args.output, 'wb') as f:
|
||||
f.write(result)
|
||||
print(f"Wrote {len(result)} bytes to {args.output}", file=sys.stderr)
|
||||
|
||||
print(f"Wrote {len(result)} bytes ({len(result)/1024:.1f} KB) to {args.output}", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
|
|
|||
36
tools/dict_compiler/download_dicts.sh
Executable file
36
tools/dict_compiler/download_dicts.sh
Executable file
|
|
@ -0,0 +1,36 @@
|
|||
#!/usr/bin/env bash
|
||||
# Download Plover and Lapwing steno dictionaries from their official repos.
|
||||
# Usage: ./download_dicts.sh [plover|lapwing|all] [output_dir]
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
THEORY="${1:-all}"
|
||||
OUTDIR="${2:-./dicts_src}"
|
||||
|
||||
mkdir -p "$OUTDIR"
|
||||
|
||||
download_plover() {
|
||||
echo "Downloading Plover main.json..."
|
||||
curl -fsSL \
|
||||
"https://raw.githubusercontent.com/openstenoproject/plover/main/plover/assets/main.json" \
|
||||
-o "$OUTDIR/plover-main.json"
|
||||
echo " $(python3 -c "import json; print(len(json.load(open('$OUTDIR/plover-main.json'))))" 2>/dev/null || echo '?') entries"
|
||||
}
|
||||
|
||||
download_lapwing() {
|
||||
local BASE="https://raw.githubusercontent.com/aerickt/plover-lapwing-aio/main/plover_lapwing/dictionaries"
|
||||
echo "Downloading Lapwing dictionaries..."
|
||||
for f in lapwing-base lapwing-commands lapwing-numbers; do
|
||||
curl -fsSL "$BASE/$f.json" -o "$OUTDIR/$f.json"
|
||||
echo " $f.json: $(python3 -c "import json; print(len(json.load(open('$OUTDIR/$f.json'))))" 2>/dev/null || echo '?') entries"
|
||||
done
|
||||
}
|
||||
|
||||
case "$THEORY" in
|
||||
plover) download_plover ;;
|
||||
lapwing) download_lapwing ;;
|
||||
all) download_plover; download_lapwing ;;
|
||||
*) echo "Usage: $0 [plover|lapwing|all] [output_dir]"; exit 1 ;;
|
||||
esac
|
||||
|
||||
echo "Done. Dictionaries saved to $OUTDIR/"
|
||||
Loading…
Add table
Add a link
Reference in a new issue