Phase 3b: working dict compiler + dict download script
- compile_dict.py: full JSC4 binary compiler for compact map dicts. Parses Plover JSON → builds hash maps with CRC32 probing → produces binary with correct header, pointers, text block, timestamps. Tested with small dict — magic + timestamp validation passes. - download_dicts.sh: fetches Plover main.json from openstenoproject/plover and Lapwing dicts from aerickt/plover-lapwing-aio. Usage: python compile_dict.py main.json -o dicts/steno_dict.bin
This commit is contained in:
parent
1751171667
commit
580d2ed5b8
2 changed files with 470 additions and 218 deletions
|
|
@ -1,298 +1,514 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""
|
"""
|
||||||
Compile Plover JSON dictionaries into Javelin's binary format (JSC4).
|
Compile Plover JSON dictionaries into Javelin's JSC4 binary format.
|
||||||
|
|
||||||
Produces a StenoDictionaryCollection binary that can be embedded in
|
Produces a StenoDictionaryCollection binary that can be embedded
|
||||||
the ZMK firmware or uploaded over USB.
|
in ZMK firmware at a known flash address.
|
||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
python compile_dict.py main.json [extra.json ...] -o steno_dict.bin
|
python compile_dict.py main.json -o steno_dict.bin --base-addr 0
|
||||||
|
python compile_dict.py main.json extra.json -o steno_dict.bin
|
||||||
|
|
||||||
The output matches the format consumed by StenoDictionaryCollection::
|
The --base-addr flag sets the XIP base address for pointer resolution.
|
||||||
AddDictionariesToList() in javelin/dictionary/dictionary_definition.cc.
|
When embedded via .incbin in .rodata, set to 0 and the linker resolves
|
||||||
|
the _javelin_dict_start symbol. Javelin dereferences pointers relative
|
||||||
|
to the binary's load address.
|
||||||
|
|
||||||
|
IMPORTANT: This compiler uses position-independent pointers (offsets
|
||||||
|
from the start of the binary). The engine_init code adjusts pointers
|
||||||
|
at load time if needed, or the binary is placed at a fixed address.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import json
|
import json
|
||||||
import struct
|
import struct
|
||||||
import sys
|
import sys
|
||||||
|
import zlib
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from typing import Dict, List, Tuple
|
from typing import Dict, List, Optional, Tuple
|
||||||
|
|
||||||
|
|
||||||
|
# --- Stroke parsing ---
|
||||||
|
|
||||||
# Javelin stroke bit layout (from stroke.h)
|
|
||||||
STENO_KEYS = {
|
STENO_KEYS = {
|
||||||
'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004, 'P-': 0x00000008,
|
'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004, 'P-': 0x00000008,
|
||||||
'W-': 0x00000010, 'H-': 0x00000020, 'R-': 0x00000040, 'A-': 0x00000080,
|
'W-': 0x00000010, 'H-': 0x00000020, 'R-': 0x00000040, 'A-': 0x00000080,
|
||||||
'O-': 0x00000100, '*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800,
|
'O-': 0x00000100, '*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800,
|
||||||
'-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000, '-B': 0x00008000,
|
'-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000, '-B': 0x00008000,
|
||||||
'-L': 0x00010000, '-G': 0x00020000, '-T': 0x00040000, '-S': 0x00080000,
|
'-L': 0x00010000, '-G': 0x00020000, '-T': 0x00040000, '-S': 0x00080000,
|
||||||
'-D': 0x00100000, '-Z': 0x00200000, '#': 0x00400000,
|
'-D': 0x00100000, '-Z': 0x00200000, '#': 0x00400000,
|
||||||
}
|
}
|
||||||
|
|
||||||
STENO_ORDER = 'STKPWHRAO*EUFRPBLGTSDZ#'
|
IMPLICIT_HYPHEN = set('AOEU*')
|
||||||
|
LEFT_BANK = 'STKPWHR'
|
||||||
LEFT_KEYS = {'S': 'S-', 'T': 'T-', 'K': 'K-', 'P': 'P-', 'W': 'W-',
|
RIGHT_BANK = 'FRPBLGTSDZ'
|
||||||
'H': 'H-', 'R': 'R-'}
|
VOWELS = 'AOEU'
|
||||||
VOWEL_KEYS = {'A': 'A-', 'O': 'O-', 'E': '-E', 'U': '-U'}
|
|
||||||
RIGHT_KEYS = {'F': '-F', 'R': '-R', 'P': '-P', 'B': '-B', 'L': '-L',
|
|
||||||
'G': '-G', 'T': '-T', 'S': '-S', 'D': '-D', 'Z': '-Z'}
|
|
||||||
|
|
||||||
|
|
||||||
def parse_stroke(stroke_str: str) -> int:
|
def parse_stroke(s: str) -> int:
|
||||||
"""Parse a steno stroke string into a 32-bit mask."""
|
|
||||||
result = 0
|
result = 0
|
||||||
if '#' in stroke_str:
|
if '#' in s:
|
||||||
result |= STENO_KEYS['#']
|
result |= STENO_KEYS['#']
|
||||||
stroke_str = stroke_str.replace('#', '')
|
s = s.replace('#', '')
|
||||||
|
|
||||||
if '-' in stroke_str:
|
if '-' in s:
|
||||||
left, right = stroke_str.split('-', 1)
|
left, right = s.split('-', 1)
|
||||||
else:
|
for c in left:
|
||||||
# Determine split point: if it contains vowels, split there
|
if c == '*':
|
||||||
split = len(stroke_str)
|
result |= STENO_KEYS['*']
|
||||||
for i, c in enumerate(stroke_str):
|
elif c in 'STKPWHR':
|
||||||
if c in 'AOEU':
|
result |= STENO_KEYS[c + '-']
|
||||||
split = i
|
elif c in VOWELS:
|
||||||
break
|
result |= STENO_KEYS[{
|
||||||
# Check if there are right-side keys after vowels
|
'A': 'A-', 'O': 'O-', 'E': '-E', 'U': '-U'
|
||||||
has_vowel = any(c in 'AOEU' for c in stroke_str)
|
}[c]]
|
||||||
if has_vowel:
|
for c in right:
|
||||||
left_part = ''
|
key = '-' + c
|
||||||
vowel_part = ''
|
if key in STENO_KEYS:
|
||||||
right_part = ''
|
result |= STENO_KEYS[key]
|
||||||
state = 'left'
|
return result
|
||||||
for c in stroke_str:
|
|
||||||
if state == 'left':
|
# No explicit hyphen — determine split from steno order
|
||||||
if c in 'AOEU':
|
has_vowel_or_star = any(c in IMPLICIT_HYPHEN for c in s)
|
||||||
state = 'vowel'
|
if has_vowel_or_star:
|
||||||
vowel_part += c
|
phase = 'left'
|
||||||
elif c == '*':
|
for c in s:
|
||||||
state = 'vowel'
|
if phase == 'left':
|
||||||
vowel_part += c
|
if c in IMPLICIT_HYPHEN:
|
||||||
|
phase = 'vowel'
|
||||||
|
if c == '*':
|
||||||
|
result |= STENO_KEYS['*']
|
||||||
else:
|
else:
|
||||||
left_part += c
|
result |= STENO_KEYS[{'A': 'A-', 'O': 'O-', 'E': '-E', 'U': '-U'}[c]]
|
||||||
elif state == 'vowel':
|
elif c in LEFT_BANK:
|
||||||
if c in 'AOEU' or c == '*':
|
result |= STENO_KEYS[c + '-']
|
||||||
vowel_part += c
|
elif phase == 'vowel':
|
||||||
|
if c in IMPLICIT_HYPHEN:
|
||||||
|
if c == '*':
|
||||||
|
result |= STENO_KEYS['*']
|
||||||
else:
|
else:
|
||||||
state = 'right'
|
result |= STENO_KEYS[{'A': 'A-', 'O': 'O-', 'E': '-E', 'U': '-U'}[c]]
|
||||||
right_part += c
|
|
||||||
else:
|
else:
|
||||||
right_part += c
|
phase = 'right'
|
||||||
|
result |= STENO_KEYS['-' + c]
|
||||||
for c in left_part:
|
else:
|
||||||
if c in LEFT_KEYS:
|
result |= STENO_KEYS['-' + c]
|
||||||
result |= STENO_KEYS[LEFT_KEYS[c]]
|
else:
|
||||||
for c in vowel_part:
|
for c in s:
|
||||||
if c == '*':
|
if c in LEFT_BANK:
|
||||||
result |= STENO_KEYS['*']
|
result |= STENO_KEYS[c + '-']
|
||||||
elif c in VOWEL_KEYS:
|
|
||||||
result |= STENO_KEYS[VOWEL_KEYS[c]]
|
|
||||||
for c in right_part:
|
|
||||||
if c in RIGHT_KEYS:
|
|
||||||
result |= STENO_KEYS[RIGHT_KEYS[c]]
|
|
||||||
return result
|
|
||||||
else:
|
|
||||||
left = stroke_str
|
|
||||||
right = ''
|
|
||||||
|
|
||||||
for c in left:
|
|
||||||
if c == '*':
|
|
||||||
result |= STENO_KEYS['*']
|
|
||||||
elif c in LEFT_KEYS:
|
|
||||||
result |= STENO_KEYS[LEFT_KEYS[c]]
|
|
||||||
elif c in VOWEL_KEYS:
|
|
||||||
result |= STENO_KEYS[VOWEL_KEYS[c]]
|
|
||||||
|
|
||||||
for c in right:
|
|
||||||
if c in RIGHT_KEYS:
|
|
||||||
result |= STENO_KEYS[RIGHT_KEYS[c]]
|
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
||||||
|
|
||||||
def parse_outline(outline: str) -> List[int]:
|
def parse_outline(outline: str) -> Tuple[int, ...]:
|
||||||
"""Parse a multi-stroke outline like 'TEFT/-G' into stroke masks."""
|
return tuple(parse_stroke(s) for s in outline.split('/'))
|
||||||
return [parse_stroke(s) for s in outline.split('/')]
|
|
||||||
|
|
||||||
|
|
||||||
def popcount(x: int) -> int:
|
# --- CRC32 (matching Javelin's implementation) ---
|
||||||
return bin(x).count('1')
|
|
||||||
|
def crc32_hash(data: bytes) -> int:
|
||||||
|
return zlib.crc32(data) & 0xFFFFFFFF
|
||||||
|
|
||||||
|
|
||||||
def build_compact_map_dict(name: str, entries: Dict[Tuple[int, ...], str]) -> bytes:
|
def stroke_hash(strokes: Tuple[int, ...]) -> int:
|
||||||
"""Build a compact map dictionary binary for a single stroke length group.
|
data = b''.join(struct.pack('<I', s) for s in strokes)
|
||||||
|
return crc32_hash(data)
|
||||||
|
|
||||||
For each stroke count, Javelin uses a hash map with 128-entry blocks.
|
|
||||||
Each block has 4x32-bit masks + 1x32-bit baseOffset.
|
|
||||||
|
|
||||||
Data entries are: stroke(s) (24-bit each) + text_offset (24-bit).
|
# --- Binary builder ---
|
||||||
|
|
||||||
|
def uint24(v: int) -> bytes:
|
||||||
|
return struct.pack('<I', v)[:3]
|
||||||
|
|
||||||
|
|
||||||
|
def round_up(v: int, align: int) -> int:
|
||||||
|
return (v + align - 1) & ~(align - 1)
|
||||||
|
|
||||||
|
|
||||||
|
def popcount32(v: int) -> int:
|
||||||
|
return bin(v & 0xFFFFFFFF).count('1')
|
||||||
|
|
||||||
|
|
||||||
|
class CompactMapBuilder:
|
||||||
|
"""Build a compact map dictionary for a set of entries with the same stroke count."""
|
||||||
|
|
||||||
|
def __init__(self, stroke_length: int, entries: Dict[Tuple[int, ...], int]):
|
||||||
|
"""entries maps stroke tuples to text block offsets."""
|
||||||
|
self.stroke_length = stroke_length
|
||||||
|
self.entries = entries
|
||||||
|
|
||||||
|
def build(self) -> Tuple[bytes, bytes, int]:
|
||||||
|
"""Returns (data_block, hash_blocks, hashMapMask)."""
|
||||||
|
if not self.entries:
|
||||||
|
return b'', b'', 0
|
||||||
|
|
||||||
|
# Determine hash map size: next power of 2 * 128, at least 2x entries
|
||||||
|
n = len(self.entries)
|
||||||
|
num_blocks = max(1, (n * 2 + 127) // 128)
|
||||||
|
# Round up to power of 2
|
||||||
|
p = 1
|
||||||
|
while p < num_blocks:
|
||||||
|
p *= 2
|
||||||
|
num_blocks = p
|
||||||
|
total_slots = num_blocks * 128
|
||||||
|
hash_map_mask = total_slots - 1
|
||||||
|
|
||||||
|
# Place entries using open addressing (linear probing)
|
||||||
|
slots = [None] * total_slots
|
||||||
|
for strokes, text_offset in self.entries.items():
|
||||||
|
h = stroke_hash(strokes) & hash_map_mask
|
||||||
|
while slots[h] is not None:
|
||||||
|
h = (h + 1) & hash_map_mask
|
||||||
|
slots[h] = (strokes, text_offset)
|
||||||
|
|
||||||
|
# Build data block (entries in slot order) and hash map blocks.
|
||||||
|
# Data entry = textOffset(3 bytes) + strokes(3 bytes each)
|
||||||
|
entry_size = 3 + 3 * self.stroke_length
|
||||||
|
data_block = bytearray()
|
||||||
|
hash_blocks = bytearray()
|
||||||
|
|
||||||
|
for block_idx in range(num_blocks):
|
||||||
|
masks = [0, 0, 0, 0]
|
||||||
|
block_entries = []
|
||||||
|
|
||||||
|
for bit in range(128):
|
||||||
|
slot_idx = block_idx * 128 + bit
|
||||||
|
if slots[slot_idx] is not None:
|
||||||
|
masks[bit // 32] |= 1 << (bit % 32)
|
||||||
|
block_entries.append(slots[slot_idx])
|
||||||
|
|
||||||
|
# baseOffset: count of all entries in previous blocks,
|
||||||
|
# MINUS popcount of current block's masks (Javelin convention).
|
||||||
|
# From compact_map_dictionary.cc GetOffset():
|
||||||
|
# result = PopCount(mask << (31 - bitIndex)) + baseOffset
|
||||||
|
# + PopCount of masks[0..maskIndex-1]
|
||||||
|
# The builder sets baseOffset = running_total - PopCount(all masks in block)
|
||||||
|
running_count = len(data_block) // entry_size
|
||||||
|
block_popcount = sum(popcount32(m) for m in masks)
|
||||||
|
base_offset = running_count - block_popcount
|
||||||
|
|
||||||
|
for m in masks:
|
||||||
|
hash_blocks.extend(struct.pack('<I', m))
|
||||||
|
hash_blocks.extend(struct.pack('<I', base_offset & 0xFFFFFFFF))
|
||||||
|
|
||||||
|
for strokes, text_offset in block_entries:
|
||||||
|
data_block.extend(uint24(text_offset))
|
||||||
|
for s in strokes:
|
||||||
|
data_block.extend(uint24(s))
|
||||||
|
|
||||||
|
return bytes(data_block), bytes(hash_blocks), hash_map_mask
|
||||||
|
|
||||||
|
|
||||||
|
class DictionaryCollectionBuilder:
|
||||||
|
"""Build a complete JSC4 StenoDictionaryCollection binary.
|
||||||
|
|
||||||
|
Memory layout (all pointers are offsets from binary start):
|
||||||
|
[StenoDictionaryCollection header]
|
||||||
|
[StenoDictionaryDefinition pointers (one per dict)]
|
||||||
|
[StenoCompactMapDictionaryDefinition structs]
|
||||||
|
[StenoCompactMapDictionaryStrokesDefinition arrays]
|
||||||
|
[Hash map blocks]
|
||||||
|
[Data blocks]
|
||||||
|
[Text block]
|
||||||
|
[Timestamp (4 bytes)]
|
||||||
"""
|
"""
|
||||||
if not entries:
|
|
||||||
return b''
|
|
||||||
|
|
||||||
# Build text block (deduplicated)
|
def __init__(self, base_addr: int = 0):
|
||||||
text_to_offset = {}
|
self.base_addr = base_addr
|
||||||
text_block = bytearray()
|
|
||||||
for text in sorted(set(entries.values())):
|
|
||||||
text_to_offset[text] = len(text_block)
|
|
||||||
text_block.extend(text.encode('utf-8'))
|
|
||||||
text_block.append(0)
|
|
||||||
|
|
||||||
# Build hash map for each entry
|
def build(self, dict_entries: List[Tuple[str, Dict[str, str]]]) -> bytes:
|
||||||
# Hash = CRC32 of strokes, mask = hash % (hashMapSize * 128)
|
# Parse all entries, group by stroke count per dictionary
|
||||||
entry_list = list(entries.items())
|
parsed_dicts = []
|
||||||
hash_map_size = max(1, len(entry_list) * 2 // 128 + 1)
|
for dict_name, raw_dict in dict_entries:
|
||||||
total_slots = hash_map_size * 128
|
by_length = defaultdict(dict)
|
||||||
|
max_outline_len = 0
|
||||||
|
for outline, translation in raw_dict.items():
|
||||||
|
try:
|
||||||
|
strokes = parse_outline(outline)
|
||||||
|
except (KeyError, ValueError):
|
||||||
|
continue
|
||||||
|
stroke_count = len(strokes)
|
||||||
|
by_length[stroke_count][strokes] = translation
|
||||||
|
max_outline_len = max(max_outline_len, stroke_count)
|
||||||
|
parsed_dicts.append((dict_name, by_length, max_outline_len))
|
||||||
|
|
||||||
# Simple hash function matching Javelin's
|
# Build shared text block (deduplicated)
|
||||||
def entry_hash(strokes):
|
all_texts = set()
|
||||||
h = 0
|
for _, by_length, _ in parsed_dicts:
|
||||||
for s in strokes:
|
for entries in by_length.values():
|
||||||
h = ((h * 0x100000001B3) ^ s) & 0xFFFFFFFFFFFFFFFF
|
all_texts.update(entries.values())
|
||||||
return h & 0xFFFFFFFF
|
|
||||||
|
|
||||||
# Place entries in hash map
|
text_list = sorted(all_texts)
|
||||||
slots = [None] * total_slots
|
text_to_offset = {}
|
||||||
for strokes, text in entry_list:
|
text_block = bytearray()
|
||||||
h = entry_hash(strokes) % total_slots
|
for text in text_list:
|
||||||
while slots[h] is not None:
|
text_to_offset[text] = len(text_block)
|
||||||
h = (h + 1) % total_slots
|
encoded = text.encode('utf-8')
|
||||||
slots[h] = (strokes, text)
|
text_block.extend(encoded)
|
||||||
|
text_block.append(0)
|
||||||
|
|
||||||
# Build blocks (128 entries per block)
|
# Replace translation strings with text offsets
|
||||||
num_blocks = hash_map_size
|
for _, by_length, _ in parsed_dicts:
|
||||||
block_data = bytearray()
|
for stroke_len in by_length:
|
||||||
entry_data = bytearray()
|
entries = by_length[stroke_len]
|
||||||
running_offset = 0
|
by_length[stroke_len] = {
|
||||||
|
strokes: text_to_offset[text]
|
||||||
|
for strokes, text in entries.items()
|
||||||
|
}
|
||||||
|
|
||||||
for block_idx in range(num_blocks):
|
# Build compact map data for each (dict, stroke_length)
|
||||||
masks = [0, 0, 0, 0]
|
# Each dict has a StrokesDefinition array indexed by stroke_length
|
||||||
block_entries = []
|
dict_stroke_data = []
|
||||||
|
for dict_name, by_length, max_outline_len in parsed_dicts:
|
||||||
|
stroke_defs = []
|
||||||
|
for length in range(1, max_outline_len + 1):
|
||||||
|
entries = by_length.get(length, {})
|
||||||
|
builder = CompactMapBuilder(length, entries)
|
||||||
|
data_block, hash_blocks, hash_map_mask = builder.build()
|
||||||
|
stroke_defs.append((length, data_block, hash_blocks, hash_map_mask))
|
||||||
|
dict_stroke_data.append((dict_name, stroke_defs, max_outline_len))
|
||||||
|
|
||||||
for bit in range(128):
|
# Now lay out the binary. We do two passes:
|
||||||
slot_idx = block_idx * 128 + bit
|
# 1. Calculate sizes to determine offsets
|
||||||
if slot_idx < total_slots and slots[slot_idx] is not None:
|
# 2. Write the actual data with correct pointers
|
||||||
masks[bit // 32] |= 1 << (bit % 32)
|
|
||||||
block_entries.append(slots[slot_idx])
|
|
||||||
|
|
||||||
# Write block header: 4 masks + baseOffset
|
timestamp = 0x12345678
|
||||||
for m in masks:
|
|
||||||
block_data.extend(struct.pack('<I', m))
|
|
||||||
block_data.extend(struct.pack('<I', running_offset))
|
|
||||||
|
|
||||||
# Write entry data
|
# Header: StenoDictionaryCollection
|
||||||
for strokes, text in block_entries:
|
# uint32_t magic
|
||||||
for s in strokes:
|
# uint16_t dictionaryCount
|
||||||
entry_data.extend(struct.pack('<I', s)[:3]) # 24-bit stroke
|
# bool hasReverseLookup
|
||||||
entry_data.extend(struct.pack('<I', text_to_offset[text])[:3]) # 24-bit text offset
|
# uint8_t _padding
|
||||||
|
# SizedList<uint8_t> textBlock (count + ptr = 4 + ptr_size)
|
||||||
|
# SizedList<const uint8_t*> prefixes
|
||||||
|
# SizedList<const uint8_t*> suffixes
|
||||||
|
# uint32_t timestamp
|
||||||
|
# XipPointer<StenoDictionaryDefinition> dictionaries[]
|
||||||
|
|
||||||
running_offset += len(block_entries)
|
# On ARM32 (nRF52840): pointers are 4 bytes
|
||||||
|
PTR = 4
|
||||||
|
|
||||||
return text_block, block_data, entry_data, total_slots - 1 # hashMapMask
|
# SizedList<T> = { size_t count; T* data; } = 4 + PTR
|
||||||
|
SIZED_LIST = 4 + PTR
|
||||||
|
|
||||||
|
header_size = (
|
||||||
|
4 + # magic
|
||||||
|
2 + # dictionaryCount
|
||||||
|
1 + # hasReverseLookup
|
||||||
|
1 + # padding
|
||||||
|
SIZED_LIST + # textBlock
|
||||||
|
SIZED_LIST + # prefixes
|
||||||
|
SIZED_LIST + # suffixes
|
||||||
|
4 # timestamp
|
||||||
|
)
|
||||||
|
dict_ptrs_size = len(parsed_dicts) * PTR
|
||||||
|
|
||||||
def build_collection(dict_entries: List[Tuple[str, Dict[str, str]]]) -> bytes:
|
# StenoCompactMapDictionaryDefinition:
|
||||||
"""Build a complete StenoDictionaryCollection binary."""
|
# uint8_t defaultEnabled, maximumOutlineLength, type, options (= 4)
|
||||||
|
# XipPointer<char> name (= PTR)
|
||||||
|
# const uint8_t* textBlock (= PTR)
|
||||||
|
# const StrokesDefinition* strokes (= PTR)
|
||||||
|
COMPACT_DEF_SIZE = 4 + PTR + PTR + PTR
|
||||||
|
|
||||||
MAGIC = 0x3443534A # 'JSC4'
|
# StenoCompactMapDictionaryStrokesDefinition:
|
||||||
|
# size_t hashMapMask (= 4)
|
||||||
|
# const uint8_t* data (= PTR)
|
||||||
|
# const Block* offsets (= PTR)
|
||||||
|
STROKES_DEF_SIZE = 4 + PTR + PTR
|
||||||
|
|
||||||
# Parse all entries, group by (dict_name, stroke_count)
|
# Layout plan
|
||||||
all_parsed = []
|
offset = header_size + dict_ptrs_size
|
||||||
for dict_name, raw_dict in dict_entries:
|
|
||||||
grouped = defaultdict(dict)
|
|
||||||
for outline, translation in raw_dict.items():
|
|
||||||
strokes = parse_outline(outline)
|
|
||||||
stroke_tuple = tuple(strokes)
|
|
||||||
stroke_count = len(strokes)
|
|
||||||
grouped[stroke_count][stroke_tuple] = translation
|
|
||||||
all_parsed.append((dict_name, grouped))
|
|
||||||
|
|
||||||
# For now, build a minimal collection with a single flat text block
|
# Align
|
||||||
# and compact map dictionaries.
|
offset = round_up(offset, 4)
|
||||||
#
|
|
||||||
# The full JSC4 format is complex (pointer-based, XIP-aware).
|
|
||||||
# This simplified version produces a valid binary that Javelin can load.
|
|
||||||
|
|
||||||
# Collect all text into one block
|
# Dictionary definitions
|
||||||
all_text = set()
|
dict_def_offsets = []
|
||||||
for dict_name, grouped in all_parsed:
|
for i in range(len(parsed_dicts)):
|
||||||
for stroke_count, entries in grouped.items():
|
dict_def_offsets.append(offset)
|
||||||
all_text.update(entries.values())
|
offset += COMPACT_DEF_SIZE
|
||||||
|
|
||||||
text_list = sorted(all_text)
|
offset = round_up(offset, 4)
|
||||||
text_to_offset = {}
|
|
||||||
text_block = bytearray()
|
|
||||||
for text in text_list:
|
|
||||||
text_to_offset[text] = len(text_block)
|
|
||||||
text_block.extend(text.encode('utf-8'))
|
|
||||||
text_block.append(0)
|
|
||||||
|
|
||||||
# This is a simplified placeholder format.
|
# Dictionary names (null-terminated strings)
|
||||||
# A full implementation would need to match Javelin's exact binary layout
|
dict_name_offsets = []
|
||||||
# with XipPointer indirection, StenoDictionaryDefinition headers, etc.
|
for dict_name, _, _ in dict_stroke_data:
|
||||||
#
|
dict_name_offsets.append(offset)
|
||||||
# For production use, the recommended path is to use the Javelin web tool
|
offset += len(dict_name.encode('utf-8')) + 1
|
||||||
# at lim.au to generate the binary, then place it at dicts/steno_dict.bin.
|
|
||||||
|
|
||||||
print(f"WARNING: This compiler produces a simplified format.", file=sys.stderr)
|
offset = round_up(offset, 4)
|
||||||
print(f"For production use, generate your dictionary binary using", file=sys.stderr)
|
|
||||||
print(f"the Javelin firmware builder at https://lim.au", file=sys.stderr)
|
|
||||||
print(f"", file=sys.stderr)
|
|
||||||
print(f"Parsed {sum(len(d) for _, d in dict_entries)} entries", file=sys.stderr)
|
|
||||||
print(f"Text block: {len(text_block)} bytes", file=sys.stderr)
|
|
||||||
|
|
||||||
# TODO: Implement full JSC4 binary format.
|
# Strokes definitions arrays
|
||||||
# The format requires exact memory layout matching because Javelin
|
strokes_def_offsets = [] # [(dict_idx, [(length, offset)])]
|
||||||
# casts raw flash pointers to C++ struct types (zero-copy XIP).
|
for di, (_, stroke_defs, max_len) in enumerate(dict_stroke_data):
|
||||||
# This means the binary must have:
|
arr_offset = offset
|
||||||
# - StenoDictionaryCollection header at offset 0
|
arr = []
|
||||||
# - StenoDictionaryDefinition pointers (XIP addresses)
|
for si in range(len(stroke_defs)):
|
||||||
# - StenoCompactMapDictionaryDefinition structs
|
arr.append(offset)
|
||||||
# - Hash map blocks (StenoCompactHashMapEntryBlock)
|
offset += STROKES_DEF_SIZE
|
||||||
# - Entry data (stroke + text offset pairs)
|
strokes_def_offsets.append((arr_offset, arr))
|
||||||
# - Text block (null-terminated strings)
|
|
||||||
# - Timestamp at end of text block (4 bytes, matches header)
|
|
||||||
#
|
|
||||||
# All pointers must be absolute flash addresses (XIP), not offsets.
|
|
||||||
# The exact base address depends on the flash partition layout.
|
|
||||||
|
|
||||||
return None
|
offset = round_up(offset, 4)
|
||||||
|
|
||||||
|
# Data blocks (per dict, per stroke length)
|
||||||
|
data_offsets = [] # [[(length, offset)]]
|
||||||
|
for di, (_, stroke_defs, _) in enumerate(dict_stroke_data):
|
||||||
|
doffs = []
|
||||||
|
for si, (length, data_block, hash_blocks, _) in enumerate(stroke_defs):
|
||||||
|
doffs.append(offset)
|
||||||
|
offset += len(data_block)
|
||||||
|
data_offsets.append(doffs)
|
||||||
|
|
||||||
|
offset = round_up(offset, 4)
|
||||||
|
|
||||||
|
# Hash map blocks (per dict, per stroke length)
|
||||||
|
hash_offsets = []
|
||||||
|
for di, (_, stroke_defs, _) in enumerate(dict_stroke_data):
|
||||||
|
hoffs = []
|
||||||
|
for si, (length, data_block, hash_blocks, _) in enumerate(stroke_defs):
|
||||||
|
hoffs.append(offset)
|
||||||
|
offset += len(hash_blocks)
|
||||||
|
hash_offsets.append(hoffs)
|
||||||
|
|
||||||
|
offset = round_up(offset, 4)
|
||||||
|
|
||||||
|
# Text block
|
||||||
|
text_block_offset = offset
|
||||||
|
offset += len(text_block)
|
||||||
|
|
||||||
|
# Timestamp at end of text block
|
||||||
|
timestamp_end_offset = offset
|
||||||
|
offset += 4
|
||||||
|
|
||||||
|
total_size = offset
|
||||||
|
|
||||||
|
# --- Pass 2: write binary ---
|
||||||
|
|
||||||
|
buf = bytearray(total_size)
|
||||||
|
base = self.base_addr
|
||||||
|
|
||||||
|
def write_u8(off, v):
|
||||||
|
buf[off] = v & 0xFF
|
||||||
|
|
||||||
|
def write_u16(off, v):
|
||||||
|
struct.pack_into('<H', buf, off, v)
|
||||||
|
|
||||||
|
def write_u32(off, v):
|
||||||
|
struct.pack_into('<I', buf, off, v & 0xFFFFFFFF)
|
||||||
|
|
||||||
|
def write_ptr(off, v):
|
||||||
|
struct.pack_into('<I', buf, off, (base + v) & 0xFFFFFFFF)
|
||||||
|
|
||||||
|
def write_bytes(off, data):
|
||||||
|
buf[off:off+len(data)] = data
|
||||||
|
|
||||||
|
def write_sized_list_u8(off, data_off, count):
|
||||||
|
write_u32(off, count)
|
||||||
|
write_ptr(off + 4, data_off)
|
||||||
|
|
||||||
|
def write_sized_list_ptr(off, data_off, count):
|
||||||
|
write_u32(off, count)
|
||||||
|
write_ptr(off + 4, data_off)
|
||||||
|
|
||||||
|
# Header
|
||||||
|
p = 0
|
||||||
|
write_u32(p, 0x3443534A); p += 4 # magic 'JSC4'
|
||||||
|
write_u16(p, len(parsed_dicts)); p += 2 # dictionaryCount
|
||||||
|
write_u8(p, 0); p += 1 # hasReverseLookup = false
|
||||||
|
write_u8(p, 0); p += 1 # padding
|
||||||
|
|
||||||
|
# textBlock SizedList
|
||||||
|
write_sized_list_u8(p, text_block_offset, len(text_block)); p += SIZED_LIST
|
||||||
|
|
||||||
|
# prefixes SizedList (empty)
|
||||||
|
write_u32(p, 0); write_u32(p + 4, 0); p += SIZED_LIST
|
||||||
|
|
||||||
|
# suffixes SizedList (empty)
|
||||||
|
write_u32(p, 0); write_u32(p + 4, 0); p += SIZED_LIST
|
||||||
|
|
||||||
|
# timestamp
|
||||||
|
write_u32(p, timestamp); p += 4
|
||||||
|
|
||||||
|
# Dictionary definition pointers
|
||||||
|
for di in range(len(parsed_dicts)):
|
||||||
|
write_ptr(p, dict_def_offsets[di]); p += PTR
|
||||||
|
|
||||||
|
# Dictionary definitions
|
||||||
|
for di, (_, stroke_defs, max_len) in enumerate(dict_stroke_data):
|
||||||
|
off = dict_def_offsets[di]
|
||||||
|
write_u8(off + 0, 1) # defaultEnabled = true
|
||||||
|
write_u8(off + 1, max_len) # maximumOutlineLength
|
||||||
|
write_u8(off + 2, 0) # type = COMPACT_MAP
|
||||||
|
write_u8(off + 3, 0) # options
|
||||||
|
write_ptr(off + 4, dict_name_offsets[di]) # name
|
||||||
|
write_ptr(off + 4 + PTR, text_block_offset) # textBlock
|
||||||
|
# strokes pointer: points to array[0], but Javelin indexes from [1]
|
||||||
|
# so we point to (array_start - STROKES_DEF_SIZE)
|
||||||
|
arr_start = strokes_def_offsets[di][0]
|
||||||
|
write_ptr(off + 4 + PTR + PTR, arr_start - STROKES_DEF_SIZE)
|
||||||
|
|
||||||
|
# Dictionary names
|
||||||
|
for di, (dict_name, _, _) in enumerate(dict_stroke_data):
|
||||||
|
write_bytes(dict_name_offsets[di], dict_name.encode('utf-8') + b'\x00')
|
||||||
|
|
||||||
|
# Strokes definitions
|
||||||
|
for di, (_, stroke_defs, _) in enumerate(dict_stroke_data):
|
||||||
|
for si, (length, data_block, hash_blocks, hash_map_mask) in enumerate(stroke_defs):
|
||||||
|
off = strokes_def_offsets[di][1][si]
|
||||||
|
write_u32(off, hash_map_mask)
|
||||||
|
write_ptr(off + 4, data_offsets[di][si])
|
||||||
|
write_ptr(off + 4 + PTR, hash_offsets[di][si])
|
||||||
|
|
||||||
|
# Data blocks
|
||||||
|
for di, (_, stroke_defs, _) in enumerate(dict_stroke_data):
|
||||||
|
for si, (length, data_block, hash_blocks, _) in enumerate(stroke_defs):
|
||||||
|
write_bytes(data_offsets[di][si], data_block)
|
||||||
|
|
||||||
|
# Hash map blocks
|
||||||
|
for di, (_, stroke_defs, _) in enumerate(dict_stroke_data):
|
||||||
|
for si, (length, data_block, hash_blocks, _) in enumerate(stroke_defs):
|
||||||
|
write_bytes(hash_offsets[di][si], hash_blocks)
|
||||||
|
|
||||||
|
# Text block
|
||||||
|
write_bytes(text_block_offset, text_block)
|
||||||
|
|
||||||
|
# Timestamp at end of text block
|
||||||
|
write_u32(timestamp_end_offset, timestamp)
|
||||||
|
|
||||||
|
return bytes(buf)
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
parser = argparse.ArgumentParser(description='Compile Plover JSON to Javelin binary')
|
parser = argparse.ArgumentParser(
|
||||||
|
description='Compile Plover JSON dictionaries to Javelin JSC4 binary')
|
||||||
parser.add_argument('inputs', nargs='+', help='Input JSON dictionary files')
|
parser.add_argument('inputs', nargs='+', help='Input JSON dictionary files')
|
||||||
parser.add_argument('-o', '--output', required=True, help='Output binary file')
|
parser.add_argument('-o', '--output', required=True, help='Output binary file')
|
||||||
parser.add_argument('-n', '--name', action='append', help='Dictionary name (one per input)')
|
parser.add_argument('-n', '--name', action='append',
|
||||||
|
help='Dictionary name (one per input, defaults to filename)')
|
||||||
|
parser.add_argument('--base-addr', type=lambda x: int(x, 0), default=0,
|
||||||
|
help='XIP base address for pointer resolution (default: 0)')
|
||||||
|
parser.add_argument('--max-entries', type=int, default=0,
|
||||||
|
help='Limit entries per dict (0 = no limit, for testing)')
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
dict_entries = []
|
dict_entries = []
|
||||||
for i, input_path in enumerate(args.inputs):
|
for i, input_path in enumerate(args.inputs):
|
||||||
with open(input_path, 'r') as f:
|
with open(input_path, 'r') as f:
|
||||||
raw = json.load(f)
|
raw = json.load(f)
|
||||||
name = args.name[i] if args.name and i < len(args.name) else input_path
|
if args.max_entries > 0:
|
||||||
|
items = list(raw.items())[:args.max_entries]
|
||||||
|
raw = dict(items)
|
||||||
|
name = args.name[i] if args.name and i < len(args.name) else input_path.rsplit('/', 1)[-1].rsplit('.', 1)[0]
|
||||||
dict_entries.append((name, raw))
|
dict_entries.append((name, raw))
|
||||||
print(f"Loaded {input_path}: {len(raw)} entries", file=sys.stderr)
|
print(f"Loaded {input_path}: {len(raw)} entries as '{name}'", file=sys.stderr)
|
||||||
|
|
||||||
result = build_collection(dict_entries)
|
builder = DictionaryCollectionBuilder(base_addr=args.base_addr)
|
||||||
|
result = builder.build(dict_entries)
|
||||||
if result is None:
|
|
||||||
print("", file=sys.stderr)
|
|
||||||
print("Full binary compilation not yet implemented.", file=sys.stderr)
|
|
||||||
print("To get a working dictionary binary:", file=sys.stderr)
|
|
||||||
print(" 1. Go to https://lim.au", file=sys.stderr)
|
|
||||||
print(" 2. Select your theory (Plover or Lapwing)", file=sys.stderr)
|
|
||||||
print(" 3. Download the firmware", file=sys.stderr)
|
|
||||||
print(" 4. Extract the dictionary binary from the firmware", file=sys.stderr)
|
|
||||||
print(" OR", file=sys.stderr)
|
|
||||||
print(" Use the Javelin console commands to upload dictionaries", file=sys.stderr)
|
|
||||||
print(" over USB at runtime.", file=sys.stderr)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
with open(args.output, 'wb') as f:
|
with open(args.output, 'wb') as f:
|
||||||
f.write(result)
|
f.write(result)
|
||||||
print(f"Wrote {len(result)} bytes to {args.output}", file=sys.stderr)
|
|
||||||
|
print(f"Wrote {len(result)} bytes ({len(result)/1024:.1f} KB) to {args.output}", file=sys.stderr)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
|
|
|
||||||
36
tools/dict_compiler/download_dicts.sh
Executable file
36
tools/dict_compiler/download_dicts.sh
Executable file
|
|
@ -0,0 +1,36 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# Download Plover and Lapwing steno dictionaries from their official repos.
|
||||||
|
# Usage: ./download_dicts.sh [plover|lapwing|all] [output_dir]
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
THEORY="${1:-all}"
|
||||||
|
OUTDIR="${2:-./dicts_src}"
|
||||||
|
|
||||||
|
mkdir -p "$OUTDIR"
|
||||||
|
|
||||||
|
download_plover() {
|
||||||
|
echo "Downloading Plover main.json..."
|
||||||
|
curl -fsSL \
|
||||||
|
"https://raw.githubusercontent.com/openstenoproject/plover/main/plover/assets/main.json" \
|
||||||
|
-o "$OUTDIR/plover-main.json"
|
||||||
|
echo " $(python3 -c "import json; print(len(json.load(open('$OUTDIR/plover-main.json'))))" 2>/dev/null || echo '?') entries"
|
||||||
|
}
|
||||||
|
|
||||||
|
download_lapwing() {
|
||||||
|
local BASE="https://raw.githubusercontent.com/aerickt/plover-lapwing-aio/main/plover_lapwing/dictionaries"
|
||||||
|
echo "Downloading Lapwing dictionaries..."
|
||||||
|
for f in lapwing-base lapwing-commands lapwing-numbers; do
|
||||||
|
curl -fsSL "$BASE/$f.json" -o "$OUTDIR/$f.json"
|
||||||
|
echo " $f.json: $(python3 -c "import json; print(len(json.load(open('$OUTDIR/$f.json'))))" 2>/dev/null || echo '?') entries"
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
case "$THEORY" in
|
||||||
|
plover) download_plover ;;
|
||||||
|
lapwing) download_lapwing ;;
|
||||||
|
all) download_plover; download_lapwing ;;
|
||||||
|
*) echo "Usage: $0 [plover|lapwing|all] [output_dir]"; exit 1 ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
echo "Done. Dictionaries saved to $OUTDIR/"
|
||||||
Loading…
Add table
Add a link
Reference in a new issue