From 234bc8d73165d585b292fd92f3a46d78ce41568e Mon Sep 17 00:00:00 2001 From: afiqzudinhadi Date: Wed, 24 Jun 2026 16:51:44 +0800 Subject: [PATCH] Initial project: benchmarks, status doc, gitignore --- .gitignore | 0 tools/alt_approaches.py | 600 +++++++++++++++++++++++++++ tools/compression_benchmark.py | 687 +++++++++++++++++++++++++++++++ tools/dawg_fst_prototype.py | 722 +++++++++++++++++++++++++++++++++ 4 files changed, 2009 insertions(+) create mode 100644 .gitignore create mode 100644 tools/alt_approaches.py create mode 100644 tools/compression_benchmark.py create mode 100644 tools/dawg_fst_prototype.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..e69de29 diff --git a/tools/alt_approaches.py b/tools/alt_approaches.py new file mode 100644 index 0000000..1f089ed --- /dev/null +++ b/tools/alt_approaches.py @@ -0,0 +1,600 @@ +#!/usr/bin/env python3 +"""Alternative compression approaches beyond DAWG/trie. + +1. Compressed page table (sorted entries, zlib pages, binary search) +2. Rule-based reduction (phonetic rules + exception table) +3. MPHF + compressed blob (no random access, decompress per-lookup) +4. Two-level hash (first stroke → bucket → scan) +5. Full compressed blob with LRU page cache +6. Hybrid combos +""" + +import json +import struct +import sys +import zlib +import math +import os +from collections import Counter, defaultdict + +STENO_KEYS = { + '#': 0x00400000, + 'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004, + 'P-': 0x00000008, 'W-': 0x00000010, 'H-': 0x00000020, + 'R-': 0x00000040, 'A-': 0x00000080, 'O-': 0x00000100, + '*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800, + '-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000, + '-B': 0x00008000, '-L': 0x00010000, '-G': 0x00020000, + '-T': 0x00040000, '-S': 0x00080000, '-D': 0x00100000, + '-Z': 0x00200000, +} +IMPLICIT_HYPHEN = set('AOEU*') + +def parse_stroke(s): + result = 0 + if '#' in s: + result |= STENO_KEYS['#'] + s = s.replace('#', '') + has_hyphen = '-' in s + s_clean = s.replace('-', '') + if not has_hyphen and not any(c in IMPLICIT_HYPHEN for c in s_clean): + for c in s_clean: + key = c + '-' + if key in STENO_KEYS: + result |= STENO_KEYS[key] + return result + past_vowels = False + for c in s_clean: + if c in 'AO': + result |= STENO_KEYS[c + '-'] + past_vowels = True + elif c in 'EU': + result |= STENO_KEYS['-' + c] + past_vowels = True + elif c == '*': + result |= STENO_KEYS['*'] + past_vowels = True + elif not past_vowels and (c + '-') in STENO_KEYS: + result |= STENO_KEYS[c + '-'] + elif past_vowels and ('-' + c) in STENO_KEYS: + result |= STENO_KEYS['-' + c] + elif has_hyphen: + if s.index(c) < s.index('-'): + result |= STENO_KEYS.get(c + '-', 0) + else: + result |= STENO_KEYS.get('-' + c, 0) + else: + if (c + '-') in STENO_KEYS: + result |= STENO_KEYS[c + '-'] + return result + + +def stroke_to_bytes(stroke_val): + """Encode stroke as 3 bytes (23 bits used).""" + return struct.pack(' DawgNode + self.final = False + self.count = 0 # reachable end nodes + + def __hash__(self): + return hash((self.final, tuple(sorted((k, v.id) for k, v in self.edges.items())))) + + def __eq__(self, other): + return (self.final == other.final and + len(self.edges) == len(other.edges) and + all(k in other.edges and self.edges[k].id == other.edges[k].id + for k in self.edges)) + +def build_dawg(entries): + """Build DAWG using Daciuk's algorithm. Entries must be sorted.""" + DawgNode.next_id = 0 + root = DawgNode() + unchecked = [] # (parent, stroke, child) + minimized = {} + prev_strokes = [] + + def minimize(down_to): + for i in range(len(unchecked) - 1, down_to - 1, -1): + parent, stroke, child = unchecked[i] + key = (child.final, tuple(sorted((k, v.id) for k, v in child.edges.items()))) + if key in minimized: + parent.edges[stroke] = minimized[key] + else: + minimized[key] = child + unchecked.pop() + + for strokes, _ in entries: + # Find common prefix + common = 0 + for i in range(min(len(strokes), len(prev_strokes))): + if strokes[i] != prev_strokes[i]: + break + common += 1 + else: + common = min(len(strokes), len(prev_strokes)) + + minimize(common) + + # Add suffix + if unchecked: + node = unchecked[-1][2] + else: + node = root + + for stroke in strokes[common:]: + new_node = DawgNode() + node.edges[stroke] = new_node + unchecked.append((node, stroke, new_node)) + node = new_node + + node.final = True + prev_strokes = strokes + + minimize(0) + + # Count reachable end nodes for each node + def count_reachable(node, visited=None): + if visited is None: + visited = {} + if node.id in visited: + return visited[node.id] + c = 1 if node.final else 0 + for child in node.edges.values(): + c += count_reachable(child, visited) + visited[node.id] = c + node.count = c + return c + + count_reachable(root) + return root + +def measure_dawg(root): + """Count nodes, edges, measure bit-packed size.""" + nodes = set() + edges = 0 + fallthrough = 0 + child_dist = Counter() + + def visit(node): + nonlocal edges, fallthrough + if node.id in nodes: + return + nodes.add(node.id) + n_children = len(node.edges) + child_dist[n_children] += 1 + edges += n_children + if n_children == 1: + fallthrough += 1 + for child in node.edges.values(): + visit(child) + + visit(root) + return len(nodes), edges, fallthrough, child_dist + +def estimate_dawg_bitpacked(n_nodes, n_edges, n_fallthrough, unique_strokes, n_entries): + """Estimate bit-packed DAWG size (smhanov format).""" + cbits = max(1, math.ceil(math.log2(max(unique_strokes, 2)))) + abits = max(1, math.ceil(math.log2(max(n_nodes, 2)))) + nskipbits = max(1, math.ceil(math.log2(max(n_entries, 2)))) + + # Fallthrough nodes: 2 + cbits bits + ft_bits = n_fallthrough * (2 + cbits) + # Leaf nodes (0 children): 2 bits + n_leaf = sum(1 for _ in range(n_nodes) if True) # approximate + # Multi-edge nodes: 2 + 1 + n_children * (cbits + nskipbits + abits) + non_ft_edges = n_edges - n_fallthrough + multi_bits = (n_nodes - n_fallthrough) * 3 # header per non-fallthrough + multi_bits += non_ft_edges * (cbits + nskipbits + abits) + + total_bits = ft_bits + multi_bits + return total_bits // 8, cbits, abits, nskipbits + + +# ─── Approach 2: MPHF + values ─── + +def estimate_mphf(n_entries, n_unique_strokes): + """Estimate MPHF-based approach size.""" + mphf_bits_per_key = 2.5 # CHD or similar + mphf_bytes = int(n_entries * mphf_bits_per_key / 8) + fingerprint_bytes = n_entries * 2 # 16-bit fingerprints + return mphf_bytes, fingerprint_bytes + + +# ─── Approach 3: LOUDS succinct trie ─── + +def build_louds_trie(entries): + """Build trie and encode as LOUDS.""" + # Build trie + class TrieNode: + __slots__ = ['children', 'is_end'] + def __init__(self): + self.children = {} + self.is_end = False + + root = TrieNode() + for strokes, _ in entries: + node = root + for s in strokes: + if s not in node.children: + node.children[s] = TrieNode() + node = node.children[s] + node.is_end = True + + # BFS to build LOUDS + from collections import deque + queue = deque([root]) + louds_bits = [] # 1 per child, 0 as separator + labels = [] + is_final = [] + n_nodes = 0 + + # Super root + louds_bits.append(1) # root is child of super root + louds_bits.append(0) + + while queue: + node = queue.popleft() + n_nodes += 1 + is_final.append(node.is_end) + for stroke in sorted(node.children.keys()): + louds_bits.append(1) + labels.append(stroke) + queue.append(node.children[stroke]) + louds_bits.append(0) # separator + + return louds_bits, labels, is_final, n_nodes + +def measure_louds(louds_bits, labels, is_final, n_nodes, unique_strokes, n_entries): + """Measure LOUDS encoding size.""" + # LOUDS bitvector + louds_bytes = (len(louds_bits) + 7) // 8 + # Rank/select auxiliary structures (~37.5% overhead for practical implementations) + rank_select_bytes = int(louds_bytes * 0.375) + # Labels: each label = stroke value + cbits = max(1, math.ceil(math.log2(max(unique_strokes, 2)))) + labels_bytes = (len(labels) * cbits + 7) // 8 + # is_final bitvector + final_bytes = (n_nodes + 7) // 8 + final_rank_bytes = int(final_bytes * 0.375) + + return { + 'louds_bitvec': louds_bytes, + 'rank_select': rank_select_bytes, + 'labels': labels_bytes, + 'is_final': final_bytes + final_rank_bytes, + 'total': louds_bytes + rank_select_bytes + labels_bytes + final_bytes + final_rank_bytes, + } + + +# ─── Approach 4: Computed entries analysis ─── + +def analyze_computed_entries(entries_dict): + """Analyze how many entries follow computable patterns.""" + computable = 0 + rule_categories = Counter() + + for stroke_str, translation in entries_dict.items(): + # Fingerspelling: single letter output from specific strokes + if len(translation) == 1 and translation.isalpha(): + computable += 1 + rule_categories['fingerspelling'] += 1 + continue + + # Number entries: output is digits + if translation.replace(',', '').replace('.', '').replace('-', '').isdigit(): + computable += 1 + rule_categories['numbers'] += 1 + continue + + # Simple suffix entries: {^ing}, {^ed}, {^ly}, {^er}, {^ment}, {^ness} + if translation.startswith('{^') and translation.endswith('}'): + suffix = translation[2:-1] + if suffix in ('ing', 'ed', 'ly', 'er', 'est', 'ment', 'ness', 'tion', + 'sion', 'able', 'ible', 'ful', 'less', 'ous', 'ive', + 'al', 'ial', 'en', 'ize', 'ise', 'ity', 'ty', + 's', 'es', "'s", 'ry', 'ary'): + computable += 1 + rule_categories['common_suffix'] += 1 + continue + + # Simple prefix entries: {pre^}, {re^}, {un^} + if translation.startswith('{') and translation.endswith('^}'): + prefix = translation[1:-2] + if prefix in ('re', 'un', 'pre', 'dis', 'mis', 'over', 'under', + 'out', 'sub', 'super', 'anti', 'auto', 'bi', 'co', + 'de', 'ex', 'inter', 'macro', 'micro', 'mid', 'mini', + 'mono', 'multi', 'non', 'post', 'semi', 'tri'): + computable += 1 + rule_categories['common_prefix'] += 1 + continue + + # Plover commands: {#...}, {PLOVER:...}, {MODE:...} + if translation.startswith('{#') or translation.startswith('{PLOVER:') or \ + translation.startswith('{MODE:'): + computable += 1 + rule_categories['commands'] += 1 + continue + + # Punctuation/formatting: {.}, {,}, {?}, {!}, {^}, {-|} + if translation in ('{.}', '{,}', '{?}', '{!}', '{^}', '{-|}', '{*-|}', + '{*!}', '{*?}', '{<}', '{>}', '{*<}', '{*>}', + '{^~|^}', '{~|}'): + computable += 1 + rule_categories['formatting'] += 1 + continue + + return computable, rule_categories + + +# ─── Approach 5: FST-style encoding ─── + +def estimate_fst(entries, unique_strokes): + """ + FST shares both prefixes AND suffixes on OUTPUT side too. + Output = sequence of output tokens along edges. + """ + # In an FST, each edge carries an output fragment + # Common output prefixes/suffixes are shared + # For steno: input = stroke sequence, output = translation + + # Build input trie first (same as DAWG input) + # Then attach output weights to edges + # FST minimization merges states with identical futures (like DAWG) + # PLUS merges output-compatible states + + # Estimate: FST typically achieves 2-4 bytes per entry for English word lists + # For steno with longer outputs, maybe 4-8 bytes per entry + + # Use BurntSushi/fst benchmarks as reference: + # 235K English words → ~750KB FST + # That's ~3.2 bytes per entry + + # For steno: 147K entries, but outputs are longer (avg 8.6 chars vs 7 for English) + # Rough: 4-6 bytes per entry + low = len(entries) * 4 + high = len(entries) * 6 + return low, high + + +# ─── Main benchmark ─── + +def main(): + # Download Plover dict + dict_path = '/tmp/plover-main.json' + if not os.path.exists(dict_path): + print("Downloading Plover main.json...") + urllib.request.urlretrieve( + "https://raw.githubusercontent.com/openstenoproject/plover/main/plover/assets/main.json", + dict_path) + + with open(dict_path) as f: + raw_dict = json.load(f) + + print(f"Plover main.json: {len(raw_dict)} entries") + print() + + # Parse all strokes + parsed = [] + unique_stroke_vals = set() + for stroke_str in sorted(raw_dict.keys()): + strokes = tuple(parse_stroke(s) for s in stroke_str.split('/')) + for s in strokes: + unique_stroke_vals.add(s) + parsed.append((strokes, raw_dict[stroke_str])) + + # Sort by stroke tuple for DAWG construction + parsed.sort(key=lambda x: x[0]) + + n_entries = len(parsed) + translations = [v for _, v in parsed] + unique_translations = set(translations) + n_unique_strokes = len(unique_stroke_vals) + + print(f"Unique stroke values: {n_unique_strokes}") + print(f"Unique translations: {len(unique_translations)}") + print() + + # ─── String table measurements ─── + print("=" * 60) + print("STRING TABLE OPTIONS") + print("=" * 60) + + raw_table, _ = build_string_table_raw(translations) + print(f" Raw deduplicated: {len(raw_table)/1024:8.1f} KB") + + fc_size, fc_count = build_string_table_front_coded(translations) + print(f" Front-coded: {fc_size/1024:8.1f} KB") + + bc_data, bc_idx, bc_cum, bc_unique = build_string_table_block_compressed( + translations, block_size=4096) + bc_total = bc_data + bc_idx + bc_cum + print(f" Block-compressed 4KB: {bc_total/1024:8.1f} KB (data={bc_data/1024:.1f} idx={bc_idx/1024:.1f})") + + bc_data2, bc_idx2, bc_cum2, _ = build_string_table_block_compressed( + translations, block_size=2048) + bc_total2 = bc_data2 + bc_idx2 + bc_cum2 + print(f" Block-compressed 2KB: {bc_total2/1024:8.1f} KB") + + bc_data3, bc_idx3, bc_cum3, _ = build_string_table_block_compressed( + translations, block_size=8192) + bc_total3 = bc_data3 + bc_idx3 + bc_cum3 + print(f" Block-compressed 8KB: {bc_total3/1024:8.1f} KB") + + # Full zlib (no random access) + full_zlib = len(zlib.compress(raw_table, 9)) + print(f" Full zlib (no RA): {full_zlib/1024:8.1f} KB") + + # Value index: maps entry index → string table position + val_idx_2b = n_entries * 2 + val_idx_3b = n_entries * 3 + # With dedup: entry → unique_string_id (17 bits for 70K) + dedup_idx_bits = n_entries * math.ceil(math.log2(len(unique_translations))) + dedup_idx_bytes = (dedup_idx_bits + 7) // 8 + print(f" Value index (2B/ent): {val_idx_2b/1024:8.1f} KB") + print(f" Value index (dedup): {dedup_idx_bytes/1024:8.1f} KB ({math.ceil(math.log2(len(unique_translations)))} bits/ent)") + print() + + # ─── Approach 1: DAWG ─── + print("=" * 60) + print("APPROACH 1: BIT-PACKED DAWG") + print("=" * 60) + + print(" Building DAWG (may take ~30s)...") + root = build_dawg(parsed) + n_nodes, n_edges, n_fallthrough, child_dist = measure_dawg(root) + print(f" Nodes: {n_nodes}") + print(f" Edges: {n_edges}") + print(f" Fallthrough (1-child): {n_fallthrough}") + print(f" Suffix dedup: {(1 - n_nodes/184582)*100:.1f}% reduction from trie") + + dawg_bytes, cbits, abits, nskipbits = estimate_dawg_bitpacked( + n_nodes, n_edges, n_fallthrough, n_unique_strokes, n_entries) + print(f" cbits={cbits} abits={abits} nskipbits={nskipbits}") + print(f" DAWG structure: {dawg_bytes/1024:8.1f} KB") + + # DAWG gives implicit index via skip counts → no separate value index needed + # Total = DAWG + string table + dawg_total = dawg_bytes + bc_total + print(f" + block-compressed strings: {bc_total/1024:.1f} KB") + print(f" TOTAL (DAWG): {dawg_total/1024:8.1f} KB") + print() + + # ─── Approach 2: MPHF ─── + print("=" * 60) + print("APPROACH 2: MPHF + BLOCK-COMPRESSED VALUES") + print("=" * 60) + + mphf_bytes, fp_bytes = estimate_mphf(n_entries, n_unique_strokes) + print(f" MPHF (~2.5 bits/key): {mphf_bytes/1024:8.1f} KB") + print(f" Fingerprints (16-bit): {fp_bytes/1024:8.1f} KB") + print(f" Fingerprints (8-bit): {fp_bytes/2/1024:8.1f} KB") + # Need to store stroke sequences for fingerprint verification + # Average stroke seq: 2.3 strokes × 3 bytes = 6.9 bytes per key + stroke_storage = int(n_entries * 2.3 * 3) + print(f" Stroke key storage: {stroke_storage/1024:8.1f} KB (for verification)") + + mphf_total_16 = mphf_bytes + fp_bytes + bc_total + dedup_idx_bytes + mphf_total_8 = mphf_bytes + fp_bytes // 2 + bc_total + dedup_idx_bytes + mphf_total_nofp = mphf_bytes + bc_total + dedup_idx_bytes # no fingerprint, accept false positives + print(f" TOTAL (16-bit fp): {mphf_total_16/1024:8.1f} KB") + print(f" TOTAL (8-bit fp): {mphf_total_8/1024:8.1f} KB") + print(f" TOTAL (no fp): {mphf_total_nofp/1024:8.1f} KB (0.4% false positive)") + print() + + # ─── Approach 3: LOUDS ─── + print("=" * 60) + print("APPROACH 3: LOUDS SUCCINCT TRIE") + print("=" * 60) + + print(" Building LOUDS trie...") + louds_bits, labels, is_final, louds_n_nodes = build_louds_trie(parsed) + louds_sizes = measure_louds(louds_bits, labels, is_final, louds_n_nodes, + n_unique_strokes, n_entries) + for k, v in louds_sizes.items(): + if k != 'total': + print(f" {k:20s}: {v/1024:8.1f} KB") + louds_total = louds_sizes['total'] + bc_total + dedup_idx_bytes + print(f" + strings + val index: {(bc_total + dedup_idx_bytes)/1024:.1f} KB") + print(f" TOTAL (LOUDS): {louds_total/1024:8.1f} KB") + print() + + # ─── Approach 4: Computed entries ─── + print("=" * 60) + print("APPROACH 4: COMPUTED ENTRIES ANALYSIS") + print("=" * 60) + + computable, categories = analyze_computed_entries(raw_dict) + remaining = n_entries - computable + print(f" Computable entries: {computable} ({computable/n_entries*100:.1f}%)") + for cat, count in categories.most_common(): + print(f" {cat:20s}: {count}") + print(f" Remaining (stored): {remaining}") + print(f" If remaining used DAWG approach:") + reduction = remaining / n_entries + computed_dawg_est = dawg_total * reduction + print(f" Estimated: {computed_dawg_est/1024:8.1f} KB") + print() + + # ─── Approach 5: FST estimate ─── + print("=" * 60) + print("APPROACH 5: FST (FINITE STATE TRANSDUCER) ESTIMATE") + print("=" * 60) + + fst_low, fst_high = estimate_fst(parsed, n_unique_strokes) + print(f" FST (4 bytes/entry): {fst_low/1024:8.1f} KB") + print(f" FST (6 bytes/entry): {fst_high/1024:8.1f} KB") + print(f" Note: FST stores keys + values together, no separate string table") + print() + + # ─── Hybrid approaches ─── + print("=" * 60) + print("HYBRID APPROACHES") + print("=" * 60) + + # Hybrid 1: Computed entries + DAWG for rest + print(f"\n HYBRID 1: Computed rules + DAWG for remaining {remaining} entries") + h1_rules = 5 # KB for rule engine code + h1_dawg = dawg_total * reduction + h1_total = h1_rules * 1024 + h1_dawg + print(f" Rules engine: {h1_rules:8.1f} KB") + print(f" DAWG (remaining): {h1_dawg/1024:8.1f} KB") + print(f" TOTAL: {h1_total/1024:8.1f} KB") + + # Hybrid 2: MPHF (no fingerprint) + full zlib strings + dedup index + print(f"\n HYBRID 2: MPHF + full zlib (decompress to RAM per-block)") + h2_total = mphf_bytes + bc_total + dedup_idx_bytes + print(f" MPHF: {mphf_bytes/1024:8.1f} KB") + print(f" Strings (block): {bc_total/1024:8.1f} KB") + print(f" Value index (dedup): {dedup_idx_bytes/1024:8.1f} KB") + print(f" TOTAL: {h2_total/1024:8.1f} KB") + + # Hybrid 3: LOUDS trie (no value index needed - use rank on is_final) + # The rank of the final-bit gives the entry index + print(f"\n HYBRID 3: LOUDS + rank-based indexing (no value index array)") + h3_total = louds_sizes['total'] + bc_total + print(f" LOUDS structure: {louds_sizes['total']/1024:8.1f} KB") + print(f" Strings (block): {bc_total/1024:8.1f} KB") + print(f" TOTAL: {h3_total/1024:8.1f} KB") + + # Hybrid 4: Computed + LOUDS for remaining + print(f"\n HYBRID 4: Computed + LOUDS for remaining {remaining}") + h4_louds_est = louds_sizes['total'] * reduction + h4_strings_est = bc_total * reduction + h4_total = h1_rules * 1024 + h4_louds_est + h4_strings_est + print(f" Rules engine: {h1_rules:8.1f} KB") + print(f" LOUDS (remaining): {h4_louds_est/1024:8.1f} KB") + print(f" Strings (remaining): {h4_strings_est/1024:8.1f} KB") + print(f" TOTAL: {h4_total/1024:8.1f} KB") + + # Hybrid 5: Computed + MPHF for remaining (no fingerprint) + print(f"\n HYBRID 5: Computed + MPHF for remaining {remaining}") + h5_mphf = int(remaining * 2.5 / 8) + h5_strings = int(bc_total * reduction) + h5_dedup = int(dedup_idx_bytes * reduction) + h5_total = h1_rules * 1024 + h5_mphf + h5_strings + h5_dedup + print(f" Rules engine: {h1_rules:8.1f} KB") + print(f" MPHF (remaining): {h5_mphf/1024:8.1f} KB") + print(f" Strings (remaining): {h5_strings/1024:8.1f} KB") + print(f" Value index: {h5_dedup/1024:8.1f} KB") + print(f" TOTAL: {h5_total/1024:8.1f} KB") + + # Hybrid 6: DAWG keys (implicit indexing) + zlib strings with smaller blocks + print(f"\n HYBRID 6: DAWG (implicit index) + aggressive string compression") + # Use DAWG skip-count for indexing (no value array) + # Try smaller zlib blocks for better compression at cost of more overhead + bc_data_1k, bc_idx_1k, bc_cum_1k, _ = build_string_table_block_compressed( + translations, block_size=1024) + bc_total_1k = bc_data_1k + bc_idx_1k + bc_cum_1k + print(f" DAWG structure: {dawg_bytes/1024:8.1f} KB") + print(f" Strings (1KB block): {bc_total_1k/1024:8.1f} KB") + print(f" TOTAL: {(dawg_bytes + bc_total_1k)/1024:8.1f} KB") + + print() + print("=" * 60) + print("SUMMARY — ALL APPROACHES RANKED BY SIZE") + print("=" * 60) + print(f" Target: 300 KB") + print() + + approaches = [ + ("DAWG + block strings", dawg_total), + ("MPHF no-fp + strings", mphf_total_nofp), + ("MPHF 8-bit fp", mphf_total_8), + ("LOUDS + strings", louds_total), + ("LOUDS rank-index", h3_total), + ("FST (optimistic)", fst_low), + ("FST (conservative)", fst_high), + ("Hybrid 1: Compute+DAWG", h1_total), + ("Hybrid 2: MPHF+block", h2_total), + ("Hybrid 3: LOUDS+rank", h3_total), + ("Hybrid 4: Compute+LOUDS", h4_total), + ("Hybrid 5: Compute+MPHF", h5_total), + ("Hybrid 6: DAWG+aggr.str", dawg_bytes + bc_total_1k), + ] + + approaches.sort(key=lambda x: x[1]) + + for name, size in approaches: + kb = size / 1024 + marker = " ✓" if kb <= 300 else f" ({kb-300:+.0f} KB over)" + print(f" {name:28s}: {kb:8.1f} KB{marker}") + + +if __name__ == '__main__': + main() diff --git a/tools/dawg_fst_prototype.py b/tools/dawg_fst_prototype.py new file mode 100644 index 0000000..9b29275 --- /dev/null +++ b/tools/dawg_fst_prototype.py @@ -0,0 +1,722 @@ +#!/usr/bin/env python3 +"""Prototype: DAWG with FST-style edge outputs. + +DAWG handles key structure (22K nodes, 87.8% suffix sharing). +Outputs encoded directly on edges — no separate value index. + +When traversing stroke sequence, accumulate output fragments from edges. +Final node's output = concatenation of all edge outputs along path. + +String dedup: outputs reference into block-compressed string table. +""" + +import json +import struct +import sys +import zlib +import math +import os +from collections import Counter, defaultdict + +# ─── Steno stroke parsing ─── + +STENO_KEYS = { + '#': 0x00400000, + 'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004, + 'P-': 0x00000008, 'W-': 0x00000010, 'H-': 0x00000020, + 'R-': 0x00000040, 'A-': 0x00000080, 'O-': 0x00000100, + '*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800, + '-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000, + '-B': 0x00008000, '-L': 0x00010000, '-G': 0x00020000, + '-T': 0x00040000, '-S': 0x00080000, '-D': 0x00100000, + '-Z': 0x00200000, +} + +IMPLICIT_HYPHEN = set('AOEU*') + +def parse_stroke(s): + result = 0 + if '#' in s: + result |= STENO_KEYS['#'] + s = s.replace('#', '') + has_hyphen = '-' in s + s_clean = s.replace('-', '') + if not has_hyphen and not any(c in IMPLICIT_HYPHEN for c in s_clean): + for c in s_clean: + key = c + '-' + if key in STENO_KEYS: + result |= STENO_KEYS[key] + return result + past_vowels = False + for c in s_clean: + if c in 'AO': + result |= STENO_KEYS[c + '-'] + past_vowels = True + elif c in 'EU': + result |= STENO_KEYS['-' + c] + past_vowels = True + elif c == '*': + result |= STENO_KEYS['*'] + past_vowels = True + elif not past_vowels and (c + '-') in STENO_KEYS: + result |= STENO_KEYS[c + '-'] + elif past_vowels and ('-' + c) in STENO_KEYS: + result |= STENO_KEYS['-' + c] + elif has_hyphen: + if s.index(c) < s.index('-'): + result |= STENO_KEYS.get(c + '-', 0) + else: + result |= STENO_KEYS.get('-' + c, 0) + else: + if (c + '-') in STENO_KEYS: + result |= STENO_KEYS[c + '-'] + return result + + +# ─── DAWG with outputs ─── + +class DawgNode: + next_id = 0 + def __init__(self): + self.id = DawgNode.next_id + DawgNode.next_id += 1 + self.edges = {} # stroke_val -> (DawgNode, output_str) + self.final = False + self.final_output = "" # remaining output at final state + + def signature(self): + """Signature for minimization — includes outputs.""" + edge_sig = tuple(sorted( + (k, child.id, out) for k, (child, out) in self.edges.items() + )) + return (self.final, self.final_output if self.final else "", edge_sig) + + def __hash__(self): + return hash(self.signature()) + + def __eq__(self, other): + return self.signature() == other.signature() + + +def build_dawg_fst(entries): + """ + Build DAWG with FST-style outputs. + + For each entry (strokes, translation): + - Walk trie path for strokes + - Attach output to FIRST edge (FST convention: push output left) + - At final node, store remaining output + + Then minimize: merge nodes with identical futures (including outputs). + + This is a simplified FST construction — not fully optimal but + captures most of the savings. + """ + DawgNode.next_id = 0 + + # Phase 1: Build trie with outputs + root = DawgNode() + + for strokes, translation in entries: + node = root + for i, stroke in enumerate(strokes): + if stroke not in node.edges: + new_node = DawgNode() + node.edges[stroke] = (new_node, "") + node = new_node + else: + node = node.edges[stroke][0] + node.final = True + node.final_output = translation + + # Phase 2: Push outputs to edges (left-push) + # For each node, find common prefix of all outputs reachable, + # push that prefix to the incoming edge, strip from descendants. + # This enables more suffix sharing. + + def push_outputs(node, depth=0): + """Push common output prefixes toward the root.""" + if not node.edges: + return + + # First recurse into children + for stroke, (child, out) in list(node.edges.items()): + push_outputs(child, depth + 1) + + # For each child, collect all outputs reachable from it + for stroke, (child, edge_out) in list(node.edges.items()): + if child.final and not child.edges: + # Leaf: output = edge_out + final_output + full_out = edge_out + child.final_output + node.edges[stroke] = (child, full_out) + child.final_output = "" + + push_outputs(root) + + # Phase 3: Minimize (merge identical subtrees including outputs) + minimized = {} + + def minimize_node(node, visited=None): + if visited is None: + visited = set() + if node.id in visited: + return node + visited.add(node.id) + + # First minimize children + for stroke, (child, out) in list(node.edges.items()): + minimized_child = minimize_node(child, visited) + node.edges[stroke] = (minimized_child, out) + + # Check if we've seen an equivalent node + sig = node.signature() + if sig in minimized: + return minimized[sig] + minimized[sig] = node + return node + + root = minimize_node(root) + + return root + + +def measure_dawg_fst(root): + """Measure the DAWG-FST structure.""" + nodes = set() + edges = 0 + total_output_bytes = 0 + output_lengths = [] + unique_outputs = set() + + def visit(node): + nonlocal edges, total_output_bytes + if node.id in nodes: + return + nodes.add(node.id) + for stroke, (child, output) in node.edges.items(): + edges += 1 + out_bytes = output.encode('utf-8') + total_output_bytes += len(out_bytes) + output_lengths.append(len(out_bytes)) + if output: + unique_outputs.add(output) + visit(child) + if node.final and node.final_output: + total_output_bytes += len(node.final_output.encode('utf-8')) + output_lengths.append(len(node.final_output.encode('utf-8'))) + unique_outputs.add(node.final_output) + + visit(root) + return { + 'nodes': len(nodes), + 'edges': edges, + 'total_output_bytes': total_output_bytes, + 'unique_outputs': len(unique_outputs), + 'avg_output_len': sum(output_lengths) / max(len(output_lengths), 1), + 'output_lengths': output_lengths, + } + + +def verify_dawg_fst(root, entries): + """Verify lookups return correct translations.""" + correct = 0 + wrong = 0 + missing = 0 + + for strokes, expected in entries: + node = root + output = "" + found = True + for stroke in strokes: + if stroke in node.edges: + child, edge_out = node.edges[stroke] + output += edge_out + node = child + else: + found = False + break + + if found and node.final: + output += node.final_output + if output == expected: + correct += 1 + else: + wrong += 1 + if wrong <= 5: + print(f" WRONG: expected '{expected}', got '{output}'") + else: + missing += 1 + if missing <= 5: + print(f" MISSING: {strokes} -> '{expected}'") + + return correct, wrong, missing + + +def estimate_binary_size(stats, n_unique_strokes): + """Estimate binary encoding size.""" + n_nodes = stats['nodes'] + n_edges = stats['edges'] + + cbits = max(1, math.ceil(math.log2(max(n_unique_strokes, 2)))) + abits = max(1, math.ceil(math.log2(max(n_nodes, 2)))) + + # Edge encoding: stroke_key + target_node + output_ref + # output_ref: index into output string table + unique_outputs = stats['unique_outputs'] + obits = max(1, math.ceil(math.log2(max(unique_outputs + 1, 2)))) # +1 for "no output" + + bits_per_edge = cbits + abits + obits + 1 # +1 for last-edge flag + total_edge_bits = n_edges * bits_per_edge + + # Node overhead: 1 bit for is_final, 1 bit for has_final_output + node_bits = n_nodes * 2 + + # Final outputs: nodes with final_output need an output reference + # Approximate: ~50% of final nodes have output + final_output_bits = n_nodes * obits // 4 # rough + + structure_bits = total_edge_bits + node_bits + final_output_bits + structure_bytes = (structure_bits + 7) // 8 + + return { + 'cbits': cbits, + 'abits': abits, + 'obits': obits, + 'bits_per_edge': bits_per_edge, + 'structure_bytes': structure_bytes, + } + + +def build_output_string_table(root): + """Collect all unique output strings and build compressed table.""" + outputs = set() + + def visit(node, visited=None): + if visited is None: + visited = set() + if node.id in visited: + return + visited.add(node.id) + for stroke, (child, output) in node.edges.items(): + if output: + outputs.add(output) + visit(child, visited) + if node.final and node.final_output: + outputs.add(node.final_output) + + visit(root) + + sorted_outputs = sorted(outputs) + + # Raw + raw = b'\x00'.join(o.encode('utf-8') for o in sorted_outputs) + raw_size = len(raw) + + # Block compressed + block_size = 4096 + blocks = [] + for i in range(0, len(raw), block_size): + block = raw[i:i+block_size] + blocks.append(zlib.compress(block, 9)) + compressed_size = sum(len(b) for b in blocks) + index_size = len(blocks) * 4 + + # Full zlib + full_zlib_size = len(zlib.compress(raw, 9)) + + return { + 'unique_count': len(sorted_outputs), + 'raw_size': raw_size, + 'block_compressed': compressed_size + index_size, + 'full_zlib': full_zlib_size, + } + + +# ─── Alternative: DAWG keys + implicit index + compressed values ─── + +def build_dawg_implicit_index(entries): + """ + Standard DAWG (no outputs on edges) but with implicit indexing. + + DAWG traversal counts reachable final nodes → gives entry index. + Entry index maps to value via simple array lookup. + + Values stored as: entry_index → string_table_offset + String table block-compressed. + + Key difference from benchmark.py: here we build actual DAWG + and measure REAL node count, then compute skip-count based size. + """ + DawgNode.next_id = 0 + root = DawgNode() + unchecked = [] + minimized = {} + prev_strokes = [] + + def minimize(down_to): + for i in range(len(unchecked) - 1, down_to - 1, -1): + parent, stroke, child = unchecked[i] + sig = (child.final, tuple(sorted((k, v.id) for k, (v, _) in child.edges.items()))) + if sig in minimized: + existing = minimized[sig] + parent.edges[stroke] = (existing, "") + else: + minimized[sig] = child + unchecked.pop() + + for strokes, translation in entries: + common = 0 + for i in range(min(len(strokes), len(prev_strokes))): + if strokes[i] != prev_strokes[i]: + break + common += 1 + else: + common = min(len(strokes), len(prev_strokes)) + + minimize(common) + + if unchecked: + node = unchecked[-1][2] + else: + node = root + + for stroke in strokes[common:]: + new_node = DawgNode() + node.edges[stroke] = (new_node, "") + unchecked.append((node, stroke, new_node)) + node = new_node + + node.final = True + prev_strokes = strokes + + minimize(0) + + # Count reachable finals for skip-count indexing + def count_finals(node, cache=None): + if cache is None: + cache = {} + if node.id in cache: + return cache[node.id] + c = 1 if node.final else 0 + for stroke in sorted(node.edges.keys()): + child, _ = node.edges[stroke] + c += count_finals(child, cache) + cache[node.id] = c + return c + + count_finals(root) + + return root + + +def verify_implicit_index(root, entries): + """Verify skip-count indexing gives correct sequential indices.""" + + def lookup_index(node, strokes): + """Return the skip-count index for a stroke sequence.""" + idx = 0 + for stroke in strokes: + # Count finals of all children with stroke < target + for s in sorted(node.edges.keys()): + if s == stroke: + child, _ = node.edges[s] + if child.final: + # This child's final state comes before its children + pass + node = child + if node.final: + idx += 1 # count this final state + break + else: + child, _ = node.edges[s] + idx += count_subtree_finals(child) + else: + return -1 + return idx - 1 if node.final else -1 + + def count_subtree_finals(node, cache={}): + if node.id in cache: + return cache[node.id] + c = 1 if node.final else 0 + for s in sorted(node.edges.keys()): + child, _ = node.edges[s] + c += count_subtree_finals(child, cache) + cache[node.id] = c + return c + + # Verify first 100 entries get sequential indices + correct = 0 + for expected_idx, (strokes, translation) in enumerate(entries[:100]): + got_idx = lookup_index(root, strokes) + if got_idx == expected_idx: + correct += 1 + + return correct + + +# ─── Main ─── + +def main(): + dict_path = '/tmp/plover-main.json' + if not os.path.exists(dict_path): + print("ERROR: Download Plover dict first:") + print(" curl -sL 'https://raw.githubusercontent.com/openstenoproject/plover/main/plover/assets/main.json' -o /tmp/plover-main.json") + sys.exit(1) + + with open(dict_path) as f: + raw_dict = json.load(f) + + # Parse and sort + parsed = [] + unique_stroke_vals = set() + for stroke_str in sorted(raw_dict.keys()): + strokes = tuple(parse_stroke(s) for s in stroke_str.split('/')) + for s in strokes: + unique_stroke_vals.add(s) + parsed.append((strokes, raw_dict[stroke_str])) + parsed.sort(key=lambda x: x[0]) + + n_entries = len(parsed) + n_unique_strokes = len(unique_stroke_vals) + translations = [v for _, v in parsed] + + print(f"Entries: {n_entries}, Unique strokes: {n_unique_strokes}") + print() + + # ─── Approach A: DAWG-FST (outputs on edges) ─── + print("=" * 60) + print("APPROACH A: DAWG-FST (outputs on edges)") + print("=" * 60) + + print("Building DAWG-FST...") + root_fst = build_dawg_fst(parsed) + + stats = measure_dawg_fst(root_fst) + print(f" Nodes: {stats['nodes']}") + print(f" Edges: {stats['edges']}") + print(f" Unique output strings: {stats['unique_outputs']}") + print(f" Total output bytes (on edges): {stats['total_output_bytes']}") + print(f" Avg output length: {stats['avg_output_len']:.1f} bytes") + + # Verify correctness + print(" Verifying lookups...") + correct, wrong, missing = verify_dawg_fst(root_fst, parsed[:1000]) + print(f" Verification (first 1000): {correct} correct, {wrong} wrong, {missing} missing") + + # Output string table + str_table = build_output_string_table(root_fst) + print(f" Output string table:") + print(f" Unique strings: {str_table['unique_count']}") + print(f" Raw: {str_table['raw_size']/1024:.1f} KB") + print(f" Block-compressed: {str_table['block_compressed']/1024:.1f} KB") + print(f" Full zlib: {str_table['full_zlib']/1024:.1f} KB") + + # Binary size estimate + bin_est = estimate_binary_size(stats, n_unique_strokes) + print(f" Binary encoding:") + print(f" cbits={bin_est['cbits']} abits={bin_est['abits']} obits={bin_est['obits']}") + print(f" Bits/edge: {bin_est['bits_per_edge']}") + print(f" Structure: {bin_est['structure_bytes']/1024:.1f} KB") + + total_a = bin_est['structure_bytes'] + str_table['block_compressed'] + print(f" TOTAL: {total_a/1024:.1f} KB") + print() + + # ─── Approach B: DAWG + implicit skip-count index ─── + print("=" * 60) + print("APPROACH B: DAWG + skip-count index (no value array)") + print("=" * 60) + + print("Building standard DAWG...") + root_std = build_dawg_implicit_index(parsed) + + stats_std = measure_dawg_fst(root_std) # reuse measurement fn + print(f" Nodes: {stats_std['nodes']}") + print(f" Edges: {stats_std['edges']}") + + # DAWG structure: edges need stroke + target + skip_count + cbits = max(1, math.ceil(math.log2(max(n_unique_strokes, 2)))) + abits = max(1, math.ceil(math.log2(max(stats_std['nodes'], 2)))) + skipbits = max(1, math.ceil(math.log2(max(n_entries, 2)))) + + # Compact: last-edge flag saves storing child count + bits_per_edge = cbits + abits + skipbits + 1 # stroke + target + skip + last_edge + structure_bits = stats_std['edges'] * bits_per_edge + stats_std['nodes'] * 1 # is_final per node + structure_bytes = (structure_bits + 7) // 8 + + print(f" cbits={cbits} abits={abits} skipbits={skipbits}") + print(f" Bits/edge: {bits_per_edge}") + print(f" Structure: {structure_bytes/1024:.1f} KB") + + # Values: skip-count gives index → look up in ordered value array + # Value array: translations in DAWG traversal order + # Need: string table + offset array (index → string table position) + unique_trans = sorted(set(translations)) + raw_strings = b'\x00'.join(t.encode('utf-8') for t in unique_trans) + + # Dedup: entry → unique_string_id + trans_to_id = {t: i for i, t in enumerate(unique_trans)} + dedup_array = [trans_to_id[t] for t in translations] + dedup_bits = math.ceil(math.log2(len(unique_trans))) + dedup_bytes = (n_entries * dedup_bits + 7) // 8 + + # String table block-compressed + blocks = [] + for i in range(0, len(raw_strings), 4096): + blocks.append(zlib.compress(raw_strings[i:i+4096], 9)) + str_compressed = sum(len(b) for b in blocks) + len(blocks) * 4 + + total_b = structure_bytes + dedup_bytes + str_compressed + print(f" Value dedup array: {dedup_bytes/1024:.1f} KB ({dedup_bits} bits/entry)") + print(f" String table: {str_compressed/1024:.1f} KB") + print(f" TOTAL: {total_b/1024:.1f} KB") + print() + + # ─── Approach C: DAWG + skip-count + NO dedup array ─── + # Instead of dedup array, store string offset directly in DAWG final nodes + print("=" * 60) + print("APPROACH C: DAWG + skip-count + direct string refs") + print("=" * 60) + + # Each final node stores a string table offset + n_final = sum(1 for _ in range(1)) # need to count + visited_c = set() + n_final_c = 0 + def count_final(node): + nonlocal n_final_c + if node.id in visited_c: + return + visited_c.add(node.id) + if node.final: + n_final_c += 1 + for s, (child, _) in node.edges.items(): + count_final(child) + count_final(root_std) + + # But wait — with DAWG suffix sharing, multiple entries share final nodes + # A shared final node can only store ONE string offset + # This breaks dedup... unless we use the skip-count to disambiguate + # Skip-count already gives unique index → use that as index into value array + # So we STILL need the value array + + # Alternative: don't share final nodes (partial DAWG — share internal only) + # Then each final node = unique entry = unique string ref + print(f" Final nodes (shared): {n_final_c}") + print(f" Total entries: {n_entries}") + print(f" Final nodes can't store unique refs with suffix sharing") + print(f" → Must use skip-count index + value array (same as Approach B)") + print() + + # ─── Approach D: Partial DAWG (share internal only) + direct refs ─── + print("=" * 60) + print("APPROACH D: Partial DAWG (no suffix sharing at finals)") + print("=" * 60) + + # Don't merge final nodes → each has unique string ref + # Merge only internal nodes + # Trade: more nodes but no value array needed + + # In the standard DAWG we had 22K nodes. + # Without suffix sharing at finals, estimate: + # 147K entries = 147K unique final nodes + shared internal nodes + # Internal nodes from DAWG: ~22K - final_shared ≈ much more nodes + # This blows up the structure. Not good. + print(f" Would need ~{n_entries} final nodes (no sharing)") + print(f" Structure would be larger than value array savings") + print(f" → Not viable") + print() + + # ─── Approach E: DAWG + varint value array ─── + print("=" * 60) + print("APPROACH E: DAWG + varint-compressed value array") + print("=" * 60) + + # Instead of fixed-width dedup_bits per entry, use varint + # Most translations are common (dedup IDs are small for frequent ones) + # Sort unique translations by frequency → frequent = small ID → small varint + + trans_freq = Counter(translations) + sorted_by_freq = sorted(set(translations), key=lambda t: -trans_freq[t]) + freq_to_id = {t: i for i, t in enumerate(sorted_by_freq)} + + # Varint encode: 7 bits per byte, high bit = continuation + def varint_size(n): + if n < 128: return 1 + if n < 16384: return 2 + if n < 2097152: return 3 + return 4 + + varint_total = sum(varint_size(freq_to_id[t]) for t in translations) + + print(f" Fixed-width value array: {dedup_bytes/1024:.1f} KB") + print(f" Varint value array: {varint_total/1024:.1f} KB") + print(f" Savings: {(dedup_bytes - varint_total)/1024:.1f} KB") + + total_e = structure_bytes + varint_total + str_compressed + print(f" TOTAL: {total_e/1024:.1f} KB") + print() + + # ─── Approach F: Hybrid — DAWG keys + Huffman-coded values ─── + print("=" * 60) + print("APPROACH F: DAWG + Huffman-coded value IDs") + print("=" * 60) + + # Huffman code the dedup IDs based on frequency + # Theoretical minimum: entropy + total_entries = len(translations) + entropy_bits = 0 + for t, count in trans_freq.items(): + p = count / total_entries + entropy_bits -= count * math.log2(p) + entropy_bytes = int(entropy_bits / 8) + + print(f" Entropy of value mapping: {entropy_bytes/1024:.1f} KB") + print(f" (theoretical minimum for value array)") + + total_f = structure_bytes + entropy_bytes + str_compressed + print(f" DAWG structure: {structure_bytes/1024:.1f} KB") + print(f" Huffman values: {entropy_bytes/1024:.1f} KB") + print(f" String table: {str_compressed/1024:.1f} KB") + print(f" TOTAL: {total_f/1024:.1f} KB") + print() + + # ─── Summary ─── + print("=" * 60) + print("SUMMARY") + print("=" * 60) + print(f" Target: 300 KB") + print() + + results = [ + ("A: DAWG-FST (edge outputs)", total_a), + ("B: DAWG + skip + dedup array", total_b), + ("E: DAWG + varint values", total_e), + ("F: DAWG + Huffman values", total_f), + ] + results.sort(key=lambda x: x[1]) + + for name, size in results: + kb = size / 1024 + marker = " ✓ FITS!" if kb <= 300 else f" ({kb-300:+.0f} KB over)" + print(f" {name:40s}: {kb:8.1f} KB{marker}") + + # Breakdown of best approach + print() + best_name, best_size = results[0] + print(f" Best: {best_name}") + print(f" Breakdown:") + if "FST" in best_name: + print(f" DAWG-FST structure: {bin_est['structure_bytes']/1024:.1f} KB") + print(f" Output string table: {str_table['block_compressed']/1024:.1f} KB") + elif "Huffman" in best_name: + print(f" DAWG structure: {structure_bytes/1024:.1f} KB") + print(f" Huffman value IDs: {entropy_bytes/1024:.1f} KB") + print(f" String table: {str_compressed/1024:.1f} KB") + elif "varint" in best_name: + print(f" DAWG structure: {structure_bytes/1024:.1f} KB") + print(f" Varint value array: {varint_total/1024:.1f} KB") + print(f" String table: {str_compressed/1024:.1f} KB") + + +if __name__ == '__main__': + main()