Initial project: benchmarks, status doc, gitignore

tools/alt_approaches.py

@@ -0,0 +1,600 @@
+#!/usr/bin/env python3
+"""Alternative compression approaches beyond DAWG/trie.
+
+1. Compressed page table (sorted entries, zlib pages, binary search)
+2. Rule-based reduction (phonetic rules + exception table)
+3. MPHF + compressed blob (no random access, decompress per-lookup)
+4. Two-level hash (first stroke → bucket → scan)
+5. Full compressed blob with LRU page cache
+6. Hybrid combos
+"""
+
+import json
+import struct
+import sys
+import zlib
+import math
+import os
+from collections import Counter, defaultdict
+
+STENO_KEYS = {
+    '#': 0x00400000,
+    'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004,
+    'P-': 0x00000008, 'W-': 0x00000010, 'H-': 0x00000020,
+    'R-': 0x00000040, 'A-': 0x00000080, 'O-': 0x00000100,
+    '*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800,
+    '-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000,
+    '-B': 0x00008000, '-L': 0x00010000, '-G': 0x00020000,
+    '-T': 0x00040000, '-S': 0x00080000, '-D': 0x00100000,
+    '-Z': 0x00200000,
+}
+IMPLICIT_HYPHEN = set('AOEU*')
+
+def parse_stroke(s):
+    result = 0
+    if '#' in s:
+        result |= STENO_KEYS['#']
+        s = s.replace('#', '')
+    has_hyphen = '-' in s
+    s_clean = s.replace('-', '')
+    if not has_hyphen and not any(c in IMPLICIT_HYPHEN for c in s_clean):
+        for c in s_clean:
+            key = c + '-'
+            if key in STENO_KEYS:
+                result |= STENO_KEYS[key]
+        return result
+    past_vowels = False
+    for c in s_clean:
+        if c in 'AO':
+            result |= STENO_KEYS[c + '-']
+            past_vowels = True
+        elif c in 'EU':
+            result |= STENO_KEYS['-' + c]
+            past_vowels = True
+        elif c == '*':
+            result |= STENO_KEYS['*']
+            past_vowels = True
+        elif not past_vowels and (c + '-') in STENO_KEYS:
+            result |= STENO_KEYS[c + '-']
+        elif past_vowels and ('-' + c) in STENO_KEYS:
+            result |= STENO_KEYS['-' + c]
+        elif has_hyphen:
+            if s.index(c) < s.index('-'):
+                result |= STENO_KEYS.get(c + '-', 0)
+            else:
+                result |= STENO_KEYS.get('-' + c, 0)
+        else:
+            if (c + '-') in STENO_KEYS:
+                result |= STENO_KEYS[c + '-']
+    return result
+
+
+def stroke_to_bytes(stroke_val):
+    """Encode stroke as 3 bytes (23 bits used)."""
+    return struct.pack('<I', stroke_val)[:3]
+
+
+# ─── Approach 1: Compressed Page Table ───
+
+def test_compressed_pages(entries, page_sizes=[64, 128, 256, 512]):
+    """
+    Sort entries, group into pages, delta-encode + zlib each page.
+    Binary search on page index for lookup.
+    """
+    print("=" * 60)
+    print("APPROACH 1: COMPRESSED PAGE TABLE")
+    print("=" * 60)
+
+    # Build string table (deduped, sorted)
+    all_translations = sorted(set(v for _, v in entries))
+    trans_to_id = {t: i for i, t in enumerate(all_translations)}
+
+    raw_strings = b'\x00'.join(t.encode('utf-8') for t in all_translations)
+    str_compressed = len(zlib.compress(raw_strings, 9))
+
+    # Block-compressed strings for random access
+    str_blocks = []
+    for i in range(0, len(raw_strings), 4096):
+        str_blocks.append(zlib.compress(raw_strings[i:i+4096], 9))
+    str_block_total = sum(len(b) for b in str_blocks) + len(str_blocks) * 4
+
+    print(f"  String table (block 4KB): {str_block_total/1024:.1f} KB")
+    print(f"  String table (full zlib): {str_compressed/1024:.1f} KB")
+    print()
+
+    for page_size in page_sizes:
+        n_pages = math.ceil(len(entries) / page_size)
+
+        # Serialize each page: delta-coded stroke sequences + translation IDs
+        page_data = []
+        page_index = []  # (first_key, compressed_offset)
+
+        total_compressed = 0
+
+        for p in range(n_pages):
+            start = p * page_size
+            end = min(start + page_size, len(entries))
+            page_entries = entries[start:end]
+
+            # First key for index
+            first_key = page_entries[0][0]
+
+            # Serialize page
+            buf = bytearray()
+            prev_strokes = ()
+
+            for strokes, translation in page_entries:
+                tid = trans_to_id[translation]
+
+                # Delta from previous: shared prefix length + new strokes + tid
+                shared = 0
+                for i in range(min(len(strokes), len(prev_strokes))):
+                    if strokes[i] == prev_strokes[i]:
+                        shared += 1
+                    else:
+                        break
+
+                # Encode: shared_len(1) + n_new(1) + new_strokes(3 each) + tid(2)
+                n_new = len(strokes) - shared
+                buf.append(shared)
+                buf.append(n_new)
+                for s in strokes[shared:]:
+                    buf.extend(stroke_to_bytes(s))
+                buf.extend(struct.pack('<H', tid & 0xFFFF))
+
+                prev_strokes = strokes
+
+            raw_page = bytes(buf)
+            compressed_page = zlib.compress(raw_page, 9)
+
+            page_index.append((first_key, total_compressed, len(compressed_page)))
+            total_compressed += len(compressed_page)
+            page_data.append(compressed_page)
+
+        # Page index size: per page = first_key (variable) + offset(3) + size(2)
+        avg_key_len = sum(len(k) * 3 for k, _, _ in page_index) / len(page_index)
+        index_entry_size = avg_key_len + 5
+        index_total = int(n_pages * index_entry_size)
+
+        # Simpler: first_key_hash(4) + offset(3) + size(2) = 9 bytes per page
+        index_simple = n_pages * 9
+
+        total = total_compressed + index_simple + str_block_total
+        total_fullzlib = total_compressed + index_simple + str_compressed
+
+        print(f"  Page size={page_size:4d}: {n_pages:4d} pages, "
+              f"pages={total_compressed/1024:.1f}KB "
+              f"idx={index_simple/1024:.1f}KB "
+              f"str={str_block_total/1024:.1f}KB "
+              f"TOTAL={total/1024:.1f}KB "
+              f"(w/full zlib str: {total_fullzlib/1024:.1f}KB)")
+
+    print()
+    return total_compressed, index_simple, str_block_total
+
+
+# ─── Approach 2: Two-Level Hash Table ───
+
+def test_two_level_hash(entries):
+    """
+    Level 1: hash(first_stroke) → bucket
+    Level 2: within bucket, linear scan of entries
+    Each bucket independently compressed.
+    """
+    print("=" * 60)
+    print("APPROACH 2: TWO-LEVEL HASH TABLE")
+    print("=" * 60)
+
+    # Group by first stroke
+    buckets = defaultdict(list)
+    for strokes, translation in entries:
+        buckets[strokes[0]].append((strokes, translation))
+
+    print(f"  Unique first strokes (buckets): {len(buckets)}")
+
+    bucket_sizes = [len(v) for v in buckets.values()]
+    print(f"  Bucket size: min={min(bucket_sizes)} max={max(bucket_sizes)} "
+          f"avg={sum(bucket_sizes)/len(bucket_sizes):.1f} "
+          f"median={sorted(bucket_sizes)[len(bucket_sizes)//2]}")
+
+    # Build string table
+    all_translations = sorted(set(v for _, v in entries))
+    trans_to_id = {t: i for i, t in enumerate(all_translations)}
+    raw_strings = b'\x00'.join(t.encode('utf-8') for t in all_translations)
+    str_blocks = []
+    for i in range(0, len(raw_strings), 4096):
+        str_blocks.append(zlib.compress(raw_strings[i:i+4096], 9))
+    str_total = sum(len(b) for b in str_blocks) + len(str_blocks) * 4
+
+    # Compress each bucket
+    total_bucket_compressed = 0
+    for first_stroke, bucket_entries in buckets.items():
+        buf = bytearray()
+        for strokes, translation in sorted(bucket_entries):
+            tid = trans_to_id[translation]
+            # remaining strokes after first
+            remaining = strokes[1:]
+            buf.append(len(remaining))
+            for s in remaining:
+                buf.extend(stroke_to_bytes(s))
+            buf.extend(struct.pack('<H', tid & 0xFFFF))
+
+        compressed = zlib.compress(bytes(buf), 9)
+        total_bucket_compressed += len(compressed)
+
+    # Bucket index: first_stroke(3) + offset(3) + size(2) = 8 bytes each
+    bucket_index = len(buckets) * 8
+
+    total = total_bucket_compressed + bucket_index + str_total
+    print(f"  Buckets compressed: {total_bucket_compressed/1024:.1f} KB")
+    print(f"  Bucket index: {bucket_index/1024:.1f} KB")
+    print(f"  String table: {str_total/1024:.1f} KB")
+    print(f"  TOTAL: {total/1024:.1f} KB")
+
+    # With full zlib strings
+    str_zlib = len(zlib.compress(raw_strings, 9))
+    total_zlib = total_bucket_compressed + bucket_index + str_zlib
+    print(f"  TOTAL (full zlib str): {total_zlib/1024:.1f} KB")
+    print()
+    return total
+
+
+# ─── Approach 3: Full zlib blob with page cache ───
+
+def test_full_compressed(entries):
+    """
+    Store everything as one zlib blob. At runtime, decompress
+    needed sections into RAM page cache. LRU eviction.
+    """
+    print("=" * 60)
+    print("APPROACH 3: FULL COMPRESSED BLOB + PAGE CACHE")
+    print("=" * 60)
+
+    all_translations = sorted(set(v for _, v in entries))
+    trans_to_id = {t: i for i, t in enumerate(all_translations)}
+
+    # Serialize all entries sorted
+    buf = bytearray()
+    prev_strokes = ()
+    for strokes, translation in entries:
+        tid = trans_to_id[translation]
+        shared = 0
+        for i in range(min(len(strokes), len(prev_strokes))):
+            if strokes[i] == prev_strokes[i]:
+                shared += 1
+            else:
+                break
+        n_new = len(strokes) - shared
+        buf.append(shared)
+        buf.append(n_new)
+        for s in strokes[shared:]:
+            buf.extend(stroke_to_bytes(s))
+        buf.extend(struct.pack('<H', tid & 0xFFFF))
+        prev_strokes = strokes
+
+    raw_entries = bytes(buf)
+
+    # String table
+    raw_strings = b'\x00'.join(t.encode('utf-8') for t in all_translations)
+
+    # Full zlib
+    entries_zlib = len(zlib.compress(raw_entries, 9))
+    strings_zlib = len(zlib.compress(raw_strings, 9))
+
+    # zstd-like (try different zlib levels and strategies)
+    entries_zlib_best = len(zlib.compress(raw_entries, 9))
+
+    print(f"  Raw entries: {len(raw_entries)/1024:.1f} KB")
+    print(f"  Raw strings: {len(raw_strings)/1024:.1f} KB")
+    print(f"  Entries zlib: {entries_zlib/1024:.1f} KB")
+    print(f"  Strings zlib: {strings_zlib/1024:.1f} KB")
+    print(f"  TOTAL (full zlib): {(entries_zlib + strings_zlib)/1024:.1f} KB")
+    print()
+    print(f"  Runtime: decompress ~4KB page per lookup (~1-5ms)")
+    print(f"  RAM cache: 8 pages × 4KB = 32KB for hot entries")
+    print()
+
+    # Sectored version: split into 4KB sectors, independently compressed
+    # Allows decompressing only the needed sector
+    sector_size = 4096
+    sectors = []
+    for i in range(0, len(raw_entries), sector_size):
+        sector = raw_entries[i:i+sector_size]
+        sectors.append(zlib.compress(sector, 9))
+
+    sectors_total = sum(len(s) for s in sectors)
+    sector_index = len(sectors) * 8  # offset(4) + first_entry_stroke_hash(4)
+
+    str_sectors = []
+    for i in range(0, len(raw_strings), sector_size):
+        str_sectors.append(zlib.compress(raw_strings[i:i+sector_size], 9))
+    str_sectors_total = sum(len(s) for s in str_sectors)
+    str_sector_index = len(str_sectors) * 4
+
+    sectored_total = sectors_total + sector_index + str_sectors_total + str_sector_index
+
+    print(f"  Sectored (4KB, random access):")
+    print(f"    Entry sectors: {sectors_total/1024:.1f} KB ({len(sectors)} sectors)")
+    print(f"    String sectors: {str_sectors_total/1024:.1f} KB ({len(str_sectors)} sectors)")
+    print(f"    Indices: {(sector_index + str_sector_index)/1024:.1f} KB")
+    print(f"    TOTAL: {sectored_total/1024:.1f} KB")
+    print()
+
+    return entries_zlib + strings_zlib, sectored_total
+
+
+# ─── Approach 4: Phonetic rule analysis ───
+
+def test_rule_reduction(raw_dict):
+    """
+    Analyze how many entries could be generated by phonetic rules.
+    Steno maps sounds → letters. Regular words follow patterns.
+    """
+    print("=" * 60)
+    print("APPROACH 4: PHONETIC RULE ANALYSIS")
+    print("=" * 60)
+
+    # Steno phonetic mappings (left hand → initial consonants)
+    LEFT_MAP = {
+        0x00000001: 's',          # S
+        0x00000002: 't',          # T
+        0x00000004: 'k',          # K
+        0x00000002|0x00000004: 'c',  # TK → d → but also 'c' context dependent
+        0x00000008: 'p',          # P
+        0x00000010: 'w',          # W
+        0x00000020: 'h',          # H
+        0x00000040: 'r',          # R
+    }
+
+    # Count single-stroke entries where output is a simple word
+    single_stroke = {k: v for k, v in raw_dict.items() if '/' not in k}
+    multi_stroke = {k: v for k, v in raw_dict.items() if '/' in k}
+
+    # Categorize translations
+    simple_words = 0  # single word, lowercase, no formatting
+    formatted = 0     # contains { } formatting
+    phrases = 0       # multiple words
+    other = 0
+
+    for translation in raw_dict.values():
+        if '{' in translation:
+            formatted += 1
+        elif ' ' in translation.strip():
+            phrases += 1
+        elif translation.strip().replace("'", "").replace("-", "").isalpha():
+            simple_words += 1
+        else:
+            other += 1
+
+    print(f"  Translation categories:")
+    print(f"    Simple words: {simple_words} ({simple_words/len(raw_dict)*100:.1f}%)")
+    print(f"    Phrases:      {phrases} ({phrases/len(raw_dict)*100:.1f}%)")
+    print(f"    Formatted:    {formatted} ({formatted/len(raw_dict)*100:.1f}%)")
+    print(f"    Other:        {other} ({other/len(raw_dict)*100:.1f}%)")
+    print()
+
+    # Suffix/prefix analysis
+    # Many multi-stroke entries = base word + suffix stroke
+    # If we store base words + suffix rules, we eliminate many entries
+
+    # Check: how many multi-stroke translations are base_word + common_suffix?
+    single_translations = set(raw_dict[k] for k in single_stroke)
+
+    common_suffixes = ['ing', 'ed', 'er', 'est', 'ly', 'ment', 'ness', 'tion',
+                       'sion', 'able', 'ible', 'ful', 'less', 'ous', 'ive',
+                       'al', 'ial', 's', 'es', "'s", 'ry', 'ary', 'ity',
+                       'ize', 'ise', 'en', 'ence', 'ance', 'ent', 'ant',
+                       'ion', 'or', 'ist', 'ism', 'ical', 'ically']
+
+    derivable = 0
+    derivable_by_suffix = Counter()
+
+    for k, v in multi_stroke.items():
+        v_clean = v.strip().lower()
+        for suffix in common_suffixes:
+            if v_clean.endswith(suffix):
+                base = v_clean[:-len(suffix)]
+                # Check variants of base in single-stroke dict
+                if base in single_translations or (base + 'e') in single_translations:
+                    derivable += 1
+                    derivable_by_suffix[suffix] += 1
+                    break
+
+    print(f"  Multi-stroke entries derivable from single + suffix:")
+    print(f"    Derivable: {derivable} of {len(multi_stroke)} ({derivable/len(multi_stroke)*100:.1f}%)")
+    for suffix, count in derivable_by_suffix.most_common(10):
+        print(f"      -{suffix}: {count}")
+
+    remaining = len(raw_dict) - derivable
+    print()
+    print(f"  If suffix rules eliminate {derivable} entries:")
+    print(f"    Remaining: {remaining} entries")
+    print(f"    Reduction: {derivable/len(raw_dict)*100:.1f}%")
+    print()
+
+    # Word frequency: many entries are for rare words
+    # English has ~3000 core words covering 95% of text
+    # How many Plover entries map to these core words?
+
+    # Rough: count unique simple output words
+    word_freq = Counter()
+    for v in raw_dict.values():
+        v_clean = v.strip().lower()
+        if v_clean.isalpha():
+            word_freq[v_clean] += 1
+
+    # Top N words cover what % of entries?
+    total_word_entries = sum(word_freq.values())
+    cumulative = 0
+    for n, (word, count) in enumerate(word_freq.most_common()):
+        cumulative += count
+        if n + 1 in (1000, 3000, 5000, 10000, 20000):
+            print(f"  Top {n+1:5d} words cover {cumulative/total_word_entries*100:.1f}% "
+                  f"of word entries ({cumulative}/{total_word_entries})")
+
+    print()
+    return remaining
+
+
+# ─── Approach 5: Multi-strategy hybrid ───
+
+def test_hybrid(entries, raw_dict):
+    """
+    Best of all approaches combined:
+    1. Phonetic rules for regular derivations (suffix combos)
+    2. Two-level hash for remaining entries
+    3. Block-compressed strings
+    4. LRU cache for hot pages
+    """
+    print("=" * 60)
+    print("APPROACH 5: MULTI-STRATEGY HYBRID")
+    print("=" * 60)
+
+    # Strategy 1: Suffix rules (from approach 4)
+    single_stroke = {k: v for k, v in raw_dict.items() if '/' not in k}
+    multi_stroke = {k: v for k, v in raw_dict.items() if '/' in k}
+    single_translations = set(raw_dict[k] for k in single_stroke)
+
+    common_suffixes = ['ing', 'ed', 'er', 'est', 'ly', 'ment', 'ness', 'tion',
+                       'sion', 'able', 'ible', 'ful', 'less', 'ous', 'ive',
+                       'al', 'ial', 's', 'es', "'s"]
+
+    suffix_derivable = set()
+    for k, v in multi_stroke.items():
+        v_clean = v.strip().lower()
+        for suffix in common_suffixes:
+            if v_clean.endswith(suffix):
+                base = v_clean[:-len(suffix)]
+                if base in single_translations or (base + 'e') in single_translations:
+                    suffix_derivable.add(k)
+                    break
+
+    # Remaining after suffix rules
+    remaining_dict = {k: v for k, v in raw_dict.items() if k not in suffix_derivable}
+    remaining_entries = []
+    for stroke_str in sorted(remaining_dict.keys()):
+        strokes = tuple(parse_stroke(s) for s in stroke_str.split('/'))
+        remaining_entries.append((strokes, remaining_dict[stroke_str]))
+    remaining_entries.sort(key=lambda x: x[0])
+
+    n_remaining = len(remaining_entries)
+
+    # Strategy 2: Compressed page table for remaining
+    all_trans = sorted(set(v for _, v in remaining_entries))
+    trans_to_id = {t: i for i, t in enumerate(all_trans)}
+    raw_strings = b'\x00'.join(t.encode('utf-8') for t in all_trans)
+
+    # String table (block-compressed)
+    str_blocks = []
+    for i in range(0, len(raw_strings), 4096):
+        str_blocks.append(zlib.compress(raw_strings[i:i+4096], 9))
+    str_total = sum(len(b) for b in str_blocks) + len(str_blocks) * 4
+
+    # Entries as compressed pages
+    page_size = 128
+    n_pages = math.ceil(n_remaining / page_size)
+    total_pages_compressed = 0
+
+    for p in range(n_pages):
+        start = p * page_size
+        end = min(start + page_size, n_remaining)
+        page = remaining_entries[start:end]
+
+        buf = bytearray()
+        prev_strokes = ()
+        for strokes, translation in page:
+            tid = trans_to_id[translation]
+            shared = 0
+            for i in range(min(len(strokes), len(prev_strokes))):
+                if strokes[i] == prev_strokes[i]:
+                    shared += 1
+                else:
+                    break
+            n_new = len(strokes) - shared
+            buf.append(shared)
+            buf.append(n_new)
+            for s in strokes[shared:]:
+                buf.extend(stroke_to_bytes(s))
+            buf.extend(struct.pack('<H', tid & 0xFFFF))
+            prev_strokes = strokes
+
+        total_pages_compressed += len(zlib.compress(bytes(buf), 9))
+
+    page_index = n_pages * 9
+
+    # Suffix rule table: store which suffix strokes map to which suffixes
+    # ~20 rules × ~8 bytes = ~160 bytes
+    suffix_rules_size = len(common_suffixes) * 8
+
+    # Base word lookup: need to know single-stroke → translation mapping
+    # This is a subset of the full dict, already included in remaining_entries
+    # (single-stroke entries are NOT removed by suffix rules)
+
+    total_hybrid = total_pages_compressed + page_index + str_total + suffix_rules_size
+
+    print(f"  Suffix rules remove: {len(suffix_derivable)} entries")
+    print(f"  Remaining entries: {n_remaining}")
+    print(f"  Components:")
+    print(f"    Suffix rules: {suffix_rules_size/1024:.2f} KB")
+    print(f"    Entry pages: {total_pages_compressed/1024:.1f} KB ({n_pages} pages)")
+    print(f"    Page index: {page_index/1024:.1f} KB")
+    print(f"    String table: {str_total/1024:.1f} KB")
+    print(f"  TOTAL: {total_hybrid/1024:.1f} KB")
+    print()
+
+    # Full zlib strings variant
+    str_zlib = len(zlib.compress(raw_strings, 9))
+    total_hybrid_zlib = total_pages_compressed + page_index + str_zlib + suffix_rules_size
+    print(f"  TOTAL (full zlib str): {total_hybrid_zlib/1024:.1f} KB")
+    print(f"    (needs ~4KB RAM for decompressing string blocks)")
+    print()
+
+    return total_hybrid, total_hybrid_zlib
+
+
+def main():
+    dict_path = '/tmp/plover-main.json'
+    with open(dict_path) as f:
+        raw_dict = json.load(f)
+
+    parsed = []
+    for stroke_str in sorted(raw_dict.keys()):
+        strokes = tuple(parse_stroke(s) for s in stroke_str.split('/'))
+        parsed.append((strokes, raw_dict[stroke_str]))
+    parsed.sort(key=lambda x: x[0])
+
+    print(f"Plover: {len(raw_dict)} entries")
+    print()
+
+    # Run all approaches
+    test_compressed_pages(parsed)
+    test_two_level_hash(parsed)
+    full_zlib, sectored = test_full_compressed(parsed)
+    remaining = test_rule_reduction(raw_dict)
+    hybrid, hybrid_zlib = test_hybrid(parsed, raw_dict)
+
+    # Summary
+    print("=" * 60)
+    print("FINAL SUMMARY — ALL APPROACHES")
+    print("=" * 60)
+    print(f"  Target: 300 KB")
+    print()
+
+    results = [
+        ("Compressed pages (128/pg)", None),  # printed inline
+        ("Two-level hash", None),
+        ("Full zlib blob", full_zlib),
+        ("Sectored (4KB)", sectored),
+        ("Hybrid (suffix rules + pages)", hybrid),
+        ("Hybrid (suffix + full zlib)", hybrid_zlib),
+    ]
+
+    for name, size in results:
+        if size:
+            kb = size / 1024
+            marker = "  ✓ FITS!" if kb <= 300 else f"  ({kb-300:+.0f} KB over)"
+            print(f"  {name:40s}: {kb:8.1f} KB{marker}")
+
+
+if __name__ == '__main__':
+    main()

tools/compression_benchmark.py

@@ -0,0 +1,687 @@
+#!/usr/bin/env python3
+"""Benchmark compression approaches for steno dictionaries.
+
+Downloads Plover main.json and measures actual output sizes for:
+1. Bit-packed DAWG (smhanov style)
+2. MPHF + block-compressed values
+3. LOUDS succinct trie
+4. Computed entries (rules + exceptions)
+5. Hybrid approaches
+"""
+
+import json
+import struct
+import sys
+import zlib
+import math
+import os
+import urllib.request
+from collections import Counter, defaultdict
+
+# ─── Steno stroke parsing ───
+
+STENO_KEYS = {
+    '#': 0x00400000,
+    'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004,
+    'P-': 0x00000008, 'W-': 0x00000010, 'H-': 0x00000020,
+    'R-': 0x00000040, 'A-': 0x00000080, 'O-': 0x00000100,
+    '*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800,
+    '-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000,
+    '-B': 0x00008000, '-L': 0x00010000, '-G': 0x00020000,
+    '-T': 0x00040000, '-S': 0x00080000, '-D': 0x00100000,
+    '-Z': 0x00200000,
+}
+
+IMPLICIT_HYPHEN = set('AOEU*')
+
+def parse_stroke(s):
+    result = 0
+    if '#' in s:
+        result |= STENO_KEYS['#']
+        s = s.replace('#', '')
+    has_hyphen = '-' in s
+    s_clean = s.replace('-', '')
+    if not has_hyphen and not any(c in IMPLICIT_HYPHEN for c in s_clean):
+        for c in s_clean:
+            key = c + '-'
+            if key in STENO_KEYS:
+                result |= STENO_KEYS[key]
+        return result
+    past_vowels = False
+    for c in s_clean:
+        if c in 'AO':
+            result |= STENO_KEYS[c + '-']
+            past_vowels = True
+        elif c in 'EU':
+            result |= STENO_KEYS['-' + c]
+            past_vowels = True
+        elif c == '*':
+            result |= STENO_KEYS['*']
+            past_vowels = True
+        elif not past_vowels and (c + '-') in STENO_KEYS:
+            result |= STENO_KEYS[c + '-']
+        elif past_vowels and ('-' + c) in STENO_KEYS:
+            result |= STENO_KEYS['-' + c]
+        elif has_hyphen:
+            if s.index(c) < s.index('-'):
+                result |= STENO_KEYS.get(c + '-', 0)
+            else:
+                result |= STENO_KEYS.get('-' + c, 0)
+        else:
+            if (c + '-') in STENO_KEYS:
+                result |= STENO_KEYS[c + '-']
+    return result
+
+
+# ─── String table compression ───
+
+def build_string_table_raw(translations):
+    """Deduplicated null-terminated strings."""
+    unique = sorted(set(translations))
+    table = b'\x00'.join(v.encode('utf-8') for v in unique)
+    idx_map = {}
+    offset = 0
+    for v in unique:
+        idx_map[v] = offset
+        offset += len(v.encode('utf-8')) + 1
+    return table, idx_map
+
+def build_string_table_block_compressed(translations, block_size=4096):
+    """Block-compressed string table with random access."""
+    unique = sorted(set(translations))
+    raw = b'\x00'.join(v.encode('utf-8') for v in unique)
+
+    blocks = []
+    block_offsets = []
+    strings_per_block = []
+    raw_offset = 0
+
+    for i in range(0, len(raw), block_size):
+        block = raw[i:i+block_size]
+        compressed = zlib.compress(block, 9)
+        block_offsets.append(len(b''.join(blocks)) if blocks else 0)
+        blocks.append(compressed)
+        count = block.count(b'\x00') + (1 if i == 0 else 0)
+        strings_per_block.append(count)
+
+    total_compressed = sum(len(b) for b in blocks)
+    index_size = len(blocks) * 4  # block offsets
+    cumulative_counts = len(blocks) * 4  # cumulative string counts
+
+    return total_compressed, index_size, cumulative_counts, len(unique)
+
+def build_string_table_front_coded(translations):
+    """Front-coded sorted strings."""
+    unique = sorted(set(translations))
+    total = 0
+    prev = b''
+    for v in unique:
+        vb = v.encode('utf-8')
+        shared = 0
+        for i in range(min(len(prev), len(vb))):
+            if prev[i] == vb[i]:
+                shared += 1
+            else:
+                break
+        total += 2 + len(vb) - shared  # prefix_len + suffix_len + suffix
+        prev = vb
+    return total, len(unique)
+
+
+# ─── Approach 1: DAWG (Daciuk algorithm) ───
+
+class DawgNode:
+    next_id = 0
+    def __init__(self):
+        self.id = DawgNode.next_id
+        DawgNode.next_id += 1
+        self.edges = {}  # stroke_value -> DawgNode
+        self.final = False
+        self.count = 0  # reachable end nodes
+
+    def __hash__(self):
+        return hash((self.final, tuple(sorted((k, v.id) for k, v in self.edges.items()))))
+
+    def __eq__(self, other):
+        return (self.final == other.final and
+                len(self.edges) == len(other.edges) and
+                all(k in other.edges and self.edges[k].id == other.edges[k].id
+                    for k in self.edges))
+
+def build_dawg(entries):
+    """Build DAWG using Daciuk's algorithm. Entries must be sorted."""
+    DawgNode.next_id = 0
+    root = DawgNode()
+    unchecked = []  # (parent, stroke, child)
+    minimized = {}
+    prev_strokes = []
+
+    def minimize(down_to):
+        for i in range(len(unchecked) - 1, down_to - 1, -1):
+            parent, stroke, child = unchecked[i]
+            key = (child.final, tuple(sorted((k, v.id) for k, v in child.edges.items())))
+            if key in minimized:
+                parent.edges[stroke] = minimized[key]
+            else:
+                minimized[key] = child
+            unchecked.pop()
+
+    for strokes, _ in entries:
+        # Find common prefix
+        common = 0
+        for i in range(min(len(strokes), len(prev_strokes))):
+            if strokes[i] != prev_strokes[i]:
+                break
+            common += 1
+        else:
+            common = min(len(strokes), len(prev_strokes))
+
+        minimize(common)
+
+        # Add suffix
+        if unchecked:
+            node = unchecked[-1][2]
+        else:
+            node = root
+
+        for stroke in strokes[common:]:
+            new_node = DawgNode()
+            node.edges[stroke] = new_node
+            unchecked.append((node, stroke, new_node))
+            node = new_node
+
+        node.final = True
+        prev_strokes = strokes
+
+    minimize(0)
+
+    # Count reachable end nodes for each node
+    def count_reachable(node, visited=None):
+        if visited is None:
+            visited = {}
+        if node.id in visited:
+            return visited[node.id]
+        c = 1 if node.final else 0
+        for child in node.edges.values():
+            c += count_reachable(child, visited)
+        visited[node.id] = c
+        node.count = c
+        return c
+
+    count_reachable(root)
+    return root
+
+def measure_dawg(root):
+    """Count nodes, edges, measure bit-packed size."""
+    nodes = set()
+    edges = 0
+    fallthrough = 0
+    child_dist = Counter()
+
+    def visit(node):
+        nonlocal edges, fallthrough
+        if node.id in nodes:
+            return
+        nodes.add(node.id)
+        n_children = len(node.edges)
+        child_dist[n_children] += 1
+        edges += n_children
+        if n_children == 1:
+            fallthrough += 1
+        for child in node.edges.values():
+            visit(child)
+
+    visit(root)
+    return len(nodes), edges, fallthrough, child_dist
+
+def estimate_dawg_bitpacked(n_nodes, n_edges, n_fallthrough, unique_strokes, n_entries):
+    """Estimate bit-packed DAWG size (smhanov format)."""
+    cbits = max(1, math.ceil(math.log2(max(unique_strokes, 2))))
+    abits = max(1, math.ceil(math.log2(max(n_nodes, 2))))
+    nskipbits = max(1, math.ceil(math.log2(max(n_entries, 2))))
+
+    # Fallthrough nodes: 2 + cbits bits
+    ft_bits = n_fallthrough * (2 + cbits)
+    # Leaf nodes (0 children): 2 bits
+    n_leaf = sum(1 for _ in range(n_nodes) if True)  # approximate
+    # Multi-edge nodes: 2 + 1 + n_children * (cbits + nskipbits + abits)
+    non_ft_edges = n_edges - n_fallthrough
+    multi_bits = (n_nodes - n_fallthrough) * 3  # header per non-fallthrough
+    multi_bits += non_ft_edges * (cbits + nskipbits + abits)
+
+    total_bits = ft_bits + multi_bits
+    return total_bits // 8, cbits, abits, nskipbits
+
+
+# ─── Approach 2: MPHF + values ───
+
+def estimate_mphf(n_entries, n_unique_strokes):
+    """Estimate MPHF-based approach size."""
+    mphf_bits_per_key = 2.5  # CHD or similar
+    mphf_bytes = int(n_entries * mphf_bits_per_key / 8)
+    fingerprint_bytes = n_entries * 2  # 16-bit fingerprints
+    return mphf_bytes, fingerprint_bytes
+
+
+# ─── Approach 3: LOUDS succinct trie ───
+
+def build_louds_trie(entries):
+    """Build trie and encode as LOUDS."""
+    # Build trie
+    class TrieNode:
+        __slots__ = ['children', 'is_end']
+        def __init__(self):
+            self.children = {}
+            self.is_end = False
+
+    root = TrieNode()
+    for strokes, _ in entries:
+        node = root
+        for s in strokes:
+            if s not in node.children:
+                node.children[s] = TrieNode()
+            node = node.children[s]
+        node.is_end = True
+
+    # BFS to build LOUDS
+    from collections import deque
+    queue = deque([root])
+    louds_bits = []  # 1 per child, 0 as separator
+    labels = []
+    is_final = []
+    n_nodes = 0
+
+    # Super root
+    louds_bits.append(1)  # root is child of super root
+    louds_bits.append(0)
+
+    while queue:
+        node = queue.popleft()
+        n_nodes += 1
+        is_final.append(node.is_end)
+        for stroke in sorted(node.children.keys()):
+            louds_bits.append(1)
+            labels.append(stroke)
+            queue.append(node.children[stroke])
+        louds_bits.append(0)  # separator
+
+    return louds_bits, labels, is_final, n_nodes
+
+def measure_louds(louds_bits, labels, is_final, n_nodes, unique_strokes, n_entries):
+    """Measure LOUDS encoding size."""
+    # LOUDS bitvector
+    louds_bytes = (len(louds_bits) + 7) // 8
+    # Rank/select auxiliary structures (~37.5% overhead for practical implementations)
+    rank_select_bytes = int(louds_bytes * 0.375)
+    # Labels: each label = stroke value
+    cbits = max(1, math.ceil(math.log2(max(unique_strokes, 2))))
+    labels_bytes = (len(labels) * cbits + 7) // 8
+    # is_final bitvector
+    final_bytes = (n_nodes + 7) // 8
+    final_rank_bytes = int(final_bytes * 0.375)
+
+    return {
+        'louds_bitvec': louds_bytes,
+        'rank_select': rank_select_bytes,
+        'labels': labels_bytes,
+        'is_final': final_bytes + final_rank_bytes,
+        'total': louds_bytes + rank_select_bytes + labels_bytes + final_bytes + final_rank_bytes,
+    }
+
+
+# ─── Approach 4: Computed entries analysis ───
+
+def analyze_computed_entries(entries_dict):
+    """Analyze how many entries follow computable patterns."""
+    computable = 0
+    rule_categories = Counter()
+
+    for stroke_str, translation in entries_dict.items():
+        # Fingerspelling: single letter output from specific strokes
+        if len(translation) == 1 and translation.isalpha():
+            computable += 1
+            rule_categories['fingerspelling'] += 1
+            continue
+
+        # Number entries: output is digits
+        if translation.replace(',', '').replace('.', '').replace('-', '').isdigit():
+            computable += 1
+            rule_categories['numbers'] += 1
+            continue
+
+        # Simple suffix entries: {^ing}, {^ed}, {^ly}, {^er}, {^ment}, {^ness}
+        if translation.startswith('{^') and translation.endswith('}'):
+            suffix = translation[2:-1]
+            if suffix in ('ing', 'ed', 'ly', 'er', 'est', 'ment', 'ness', 'tion',
+                         'sion', 'able', 'ible', 'ful', 'less', 'ous', 'ive',
+                         'al', 'ial', 'en', 'ize', 'ise', 'ity', 'ty',
+                         's', 'es', "'s", 'ry', 'ary'):
+                computable += 1
+                rule_categories['common_suffix'] += 1
+                continue
+
+        # Simple prefix entries: {pre^}, {re^}, {un^}
+        if translation.startswith('{') and translation.endswith('^}'):
+            prefix = translation[1:-2]
+            if prefix in ('re', 'un', 'pre', 'dis', 'mis', 'over', 'under',
+                         'out', 'sub', 'super', 'anti', 'auto', 'bi', 'co',
+                         'de', 'ex', 'inter', 'macro', 'micro', 'mid', 'mini',
+                         'mono', 'multi', 'non', 'post', 'semi', 'tri'):
+                computable += 1
+                rule_categories['common_prefix'] += 1
+                continue
+
+        # Plover commands: {#...}, {PLOVER:...}, {MODE:...}
+        if translation.startswith('{#') or translation.startswith('{PLOVER:') or \
+           translation.startswith('{MODE:'):
+            computable += 1
+            rule_categories['commands'] += 1
+            continue
+
+        # Punctuation/formatting: {.}, {,}, {?}, {!}, {^}, {-|}
+        if translation in ('{.}', '{,}', '{?}', '{!}', '{^}', '{-|}', '{*-|}',
+                          '{*!}', '{*?}', '{<}', '{>}', '{*<}', '{*>}',
+                          '{^~|^}', '{~|}'):
+            computable += 1
+            rule_categories['formatting'] += 1
+            continue
+
+    return computable, rule_categories
+
+
+# ─── Approach 5: FST-style encoding ───
+
+def estimate_fst(entries, unique_strokes):
+    """
+    FST shares both prefixes AND suffixes on OUTPUT side too.
+    Output = sequence of output tokens along edges.
+    """
+    # In an FST, each edge carries an output fragment
+    # Common output prefixes/suffixes are shared
+    # For steno: input = stroke sequence, output = translation
+
+    # Build input trie first (same as DAWG input)
+    # Then attach output weights to edges
+    # FST minimization merges states with identical futures (like DAWG)
+    # PLUS merges output-compatible states
+
+    # Estimate: FST typically achieves 2-4 bytes per entry for English word lists
+    # For steno with longer outputs, maybe 4-8 bytes per entry
+
+    # Use BurntSushi/fst benchmarks as reference:
+    # 235K English words → ~750KB FST
+    # That's ~3.2 bytes per entry
+
+    # For steno: 147K entries, but outputs are longer (avg 8.6 chars vs 7 for English)
+    # Rough: 4-6 bytes per entry
+    low = len(entries) * 4
+    high = len(entries) * 6
+    return low, high
+
+
+# ─── Main benchmark ───
+
+def main():
+    # Download Plover dict
+    dict_path = '/tmp/plover-main.json'
+    if not os.path.exists(dict_path):
+        print("Downloading Plover main.json...")
+        urllib.request.urlretrieve(
+            "https://raw.githubusercontent.com/openstenoproject/plover/main/plover/assets/main.json",
+            dict_path)
+
+    with open(dict_path) as f:
+        raw_dict = json.load(f)
+
+    print(f"Plover main.json: {len(raw_dict)} entries")
+    print()
+
+    # Parse all strokes
+    parsed = []
+    unique_stroke_vals = set()
+    for stroke_str in sorted(raw_dict.keys()):
+        strokes = tuple(parse_stroke(s) for s in stroke_str.split('/'))
+        for s in strokes:
+            unique_stroke_vals.add(s)
+        parsed.append((strokes, raw_dict[stroke_str]))
+
+    # Sort by stroke tuple for DAWG construction
+    parsed.sort(key=lambda x: x[0])
+
+    n_entries = len(parsed)
+    translations = [v for _, v in parsed]
+    unique_translations = set(translations)
+    n_unique_strokes = len(unique_stroke_vals)
+
+    print(f"Unique stroke values: {n_unique_strokes}")
+    print(f"Unique translations: {len(unique_translations)}")
+    print()
+
+    # ─── String table measurements ───
+    print("=" * 60)
+    print("STRING TABLE OPTIONS")
+    print("=" * 60)
+
+    raw_table, _ = build_string_table_raw(translations)
+    print(f"  Raw deduplicated:      {len(raw_table)/1024:8.1f} KB")
+
+    fc_size, fc_count = build_string_table_front_coded(translations)
+    print(f"  Front-coded:           {fc_size/1024:8.1f} KB")
+
+    bc_data, bc_idx, bc_cum, bc_unique = build_string_table_block_compressed(
+        translations, block_size=4096)
+    bc_total = bc_data + bc_idx + bc_cum
+    print(f"  Block-compressed 4KB:  {bc_total/1024:8.1f} KB  (data={bc_data/1024:.1f} idx={bc_idx/1024:.1f})")
+
+    bc_data2, bc_idx2, bc_cum2, _ = build_string_table_block_compressed(
+        translations, block_size=2048)
+    bc_total2 = bc_data2 + bc_idx2 + bc_cum2
+    print(f"  Block-compressed 2KB:  {bc_total2/1024:8.1f} KB")
+
+    bc_data3, bc_idx3, bc_cum3, _ = build_string_table_block_compressed(
+        translations, block_size=8192)
+    bc_total3 = bc_data3 + bc_idx3 + bc_cum3
+    print(f"  Block-compressed 8KB:  {bc_total3/1024:8.1f} KB")
+
+    # Full zlib (no random access)
+    full_zlib = len(zlib.compress(raw_table, 9))
+    print(f"  Full zlib (no RA):     {full_zlib/1024:8.1f} KB")
+
+    # Value index: maps entry index → string table position
+    val_idx_2b = n_entries * 2
+    val_idx_3b = n_entries * 3
+    # With dedup: entry → unique_string_id (17 bits for 70K)
+    dedup_idx_bits = n_entries * math.ceil(math.log2(len(unique_translations)))
+    dedup_idx_bytes = (dedup_idx_bits + 7) // 8
+    print(f"  Value index (2B/ent):  {val_idx_2b/1024:8.1f} KB")
+    print(f"  Value index (dedup):   {dedup_idx_bytes/1024:8.1f} KB  ({math.ceil(math.log2(len(unique_translations)))} bits/ent)")
+    print()
+
+    # ─── Approach 1: DAWG ───
+    print("=" * 60)
+    print("APPROACH 1: BIT-PACKED DAWG")
+    print("=" * 60)
+
+    print("  Building DAWG (may take ~30s)...")
+    root = build_dawg(parsed)
+    n_nodes, n_edges, n_fallthrough, child_dist = measure_dawg(root)
+    print(f"  Nodes: {n_nodes}")
+    print(f"  Edges: {n_edges}")
+    print(f"  Fallthrough (1-child): {n_fallthrough}")
+    print(f"  Suffix dedup: {(1 - n_nodes/184582)*100:.1f}% reduction from trie")
+
+    dawg_bytes, cbits, abits, nskipbits = estimate_dawg_bitpacked(
+        n_nodes, n_edges, n_fallthrough, n_unique_strokes, n_entries)
+    print(f"  cbits={cbits} abits={abits} nskipbits={nskipbits}")
+    print(f"  DAWG structure:        {dawg_bytes/1024:8.1f} KB")
+
+    # DAWG gives implicit index via skip counts → no separate value index needed
+    # Total = DAWG + string table
+    dawg_total = dawg_bytes + bc_total
+    print(f"  + block-compressed strings: {bc_total/1024:.1f} KB")
+    print(f"  TOTAL (DAWG):          {dawg_total/1024:8.1f} KB")
+    print()
+
+    # ─── Approach 2: MPHF ───
+    print("=" * 60)
+    print("APPROACH 2: MPHF + BLOCK-COMPRESSED VALUES")
+    print("=" * 60)
+
+    mphf_bytes, fp_bytes = estimate_mphf(n_entries, n_unique_strokes)
+    print(f"  MPHF (~2.5 bits/key):  {mphf_bytes/1024:8.1f} KB")
+    print(f"  Fingerprints (16-bit): {fp_bytes/1024:8.1f} KB")
+    print(f"  Fingerprints (8-bit):  {fp_bytes/2/1024:8.1f} KB")
+    # Need to store stroke sequences for fingerprint verification
+    # Average stroke seq: 2.3 strokes × 3 bytes = 6.9 bytes per key
+    stroke_storage = int(n_entries * 2.3 * 3)
+    print(f"  Stroke key storage:    {stroke_storage/1024:8.1f} KB  (for verification)")
+
+    mphf_total_16 = mphf_bytes + fp_bytes + bc_total + dedup_idx_bytes
+    mphf_total_8 = mphf_bytes + fp_bytes // 2 + bc_total + dedup_idx_bytes
+    mphf_total_nofp = mphf_bytes + bc_total + dedup_idx_bytes  # no fingerprint, accept false positives
+    print(f"  TOTAL (16-bit fp):     {mphf_total_16/1024:8.1f} KB")
+    print(f"  TOTAL (8-bit fp):      {mphf_total_8/1024:8.1f} KB")
+    print(f"  TOTAL (no fp):         {mphf_total_nofp/1024:8.1f} KB  (0.4% false positive)")
+    print()
+
+    # ─── Approach 3: LOUDS ───
+    print("=" * 60)
+    print("APPROACH 3: LOUDS SUCCINCT TRIE")
+    print("=" * 60)
+
+    print("  Building LOUDS trie...")
+    louds_bits, labels, is_final, louds_n_nodes = build_louds_trie(parsed)
+    louds_sizes = measure_louds(louds_bits, labels, is_final, louds_n_nodes,
+                                 n_unique_strokes, n_entries)
+    for k, v in louds_sizes.items():
+        if k != 'total':
+            print(f"  {k:20s}:  {v/1024:8.1f} KB")
+    louds_total = louds_sizes['total'] + bc_total + dedup_idx_bytes
+    print(f"  + strings + val index: {(bc_total + dedup_idx_bytes)/1024:.1f} KB")
+    print(f"  TOTAL (LOUDS):         {louds_total/1024:8.1f} KB")
+    print()
+
+    # ─── Approach 4: Computed entries ───
+    print("=" * 60)
+    print("APPROACH 4: COMPUTED ENTRIES ANALYSIS")
+    print("=" * 60)
+
+    computable, categories = analyze_computed_entries(raw_dict)
+    remaining = n_entries - computable
+    print(f"  Computable entries:    {computable} ({computable/n_entries*100:.1f}%)")
+    for cat, count in categories.most_common():
+        print(f"    {cat:20s}: {count}")
+    print(f"  Remaining (stored):    {remaining}")
+    print(f"  If remaining used DAWG approach:")
+    reduction = remaining / n_entries
+    computed_dawg_est = dawg_total * reduction
+    print(f"    Estimated:           {computed_dawg_est/1024:8.1f} KB")
+    print()
+
+    # ─── Approach 5: FST estimate ───
+    print("=" * 60)
+    print("APPROACH 5: FST (FINITE STATE TRANSDUCER) ESTIMATE")
+    print("=" * 60)
+
+    fst_low, fst_high = estimate_fst(parsed, n_unique_strokes)
+    print(f"  FST (4 bytes/entry):   {fst_low/1024:8.1f} KB")
+    print(f"  FST (6 bytes/entry):   {fst_high/1024:8.1f} KB")
+    print(f"  Note: FST stores keys + values together, no separate string table")
+    print()
+
+    # ─── Hybrid approaches ───
+    print("=" * 60)
+    print("HYBRID APPROACHES")
+    print("=" * 60)
+
+    # Hybrid 1: Computed entries + DAWG for rest
+    print(f"\n  HYBRID 1: Computed rules + DAWG for remaining {remaining} entries")
+    h1_rules = 5  # KB for rule engine code
+    h1_dawg = dawg_total * reduction
+    h1_total = h1_rules * 1024 + h1_dawg
+    print(f"    Rules engine:        {h1_rules:8.1f} KB")
+    print(f"    DAWG (remaining):    {h1_dawg/1024:8.1f} KB")
+    print(f"    TOTAL:               {h1_total/1024:8.1f} KB")
+
+    # Hybrid 2: MPHF (no fingerprint) + full zlib strings + dedup index
+    print(f"\n  HYBRID 2: MPHF + full zlib (decompress to RAM per-block)")
+    h2_total = mphf_bytes + bc_total + dedup_idx_bytes
+    print(f"    MPHF:                {mphf_bytes/1024:8.1f} KB")
+    print(f"    Strings (block):     {bc_total/1024:8.1f} KB")
+    print(f"    Value index (dedup): {dedup_idx_bytes/1024:8.1f} KB")
+    print(f"    TOTAL:               {h2_total/1024:8.1f} KB")
+
+    # Hybrid 3: LOUDS trie (no value index needed - use rank on is_final)
+    # The rank of the final-bit gives the entry index
+    print(f"\n  HYBRID 3: LOUDS + rank-based indexing (no value index array)")
+    h3_total = louds_sizes['total'] + bc_total
+    print(f"    LOUDS structure:     {louds_sizes['total']/1024:8.1f} KB")
+    print(f"    Strings (block):     {bc_total/1024:8.1f} KB")
+    print(f"    TOTAL:               {h3_total/1024:8.1f} KB")
+
+    # Hybrid 4: Computed + LOUDS for remaining
+    print(f"\n  HYBRID 4: Computed + LOUDS for remaining {remaining}")
+    h4_louds_est = louds_sizes['total'] * reduction
+    h4_strings_est = bc_total * reduction
+    h4_total = h1_rules * 1024 + h4_louds_est + h4_strings_est
+    print(f"    Rules engine:        {h1_rules:8.1f} KB")
+    print(f"    LOUDS (remaining):   {h4_louds_est/1024:8.1f} KB")
+    print(f"    Strings (remaining): {h4_strings_est/1024:8.1f} KB")
+    print(f"    TOTAL:               {h4_total/1024:8.1f} KB")
+
+    # Hybrid 5: Computed + MPHF for remaining (no fingerprint)
+    print(f"\n  HYBRID 5: Computed + MPHF for remaining {remaining}")
+    h5_mphf = int(remaining * 2.5 / 8)
+    h5_strings = int(bc_total * reduction)
+    h5_dedup = int(dedup_idx_bytes * reduction)
+    h5_total = h1_rules * 1024 + h5_mphf + h5_strings + h5_dedup
+    print(f"    Rules engine:        {h1_rules:8.1f} KB")
+    print(f"    MPHF (remaining):    {h5_mphf/1024:8.1f} KB")
+    print(f"    Strings (remaining): {h5_strings/1024:8.1f} KB")
+    print(f"    Value index:         {h5_dedup/1024:8.1f} KB")
+    print(f"    TOTAL:               {h5_total/1024:8.1f} KB")
+
+    # Hybrid 6: DAWG keys (implicit indexing) + zlib strings with smaller blocks
+    print(f"\n  HYBRID 6: DAWG (implicit index) + aggressive string compression")
+    # Use DAWG skip-count for indexing (no value array)
+    # Try smaller zlib blocks for better compression at cost of more overhead
+    bc_data_1k, bc_idx_1k, bc_cum_1k, _ = build_string_table_block_compressed(
+        translations, block_size=1024)
+    bc_total_1k = bc_data_1k + bc_idx_1k + bc_cum_1k
+    print(f"    DAWG structure:      {dawg_bytes/1024:8.1f} KB")
+    print(f"    Strings (1KB block): {bc_total_1k/1024:8.1f} KB")
+    print(f"    TOTAL:               {(dawg_bytes + bc_total_1k)/1024:8.1f} KB")
+
+    print()
+    print("=" * 60)
+    print("SUMMARY — ALL APPROACHES RANKED BY SIZE")
+    print("=" * 60)
+    print(f"  Target: 300 KB")
+    print()
+
+    approaches = [
+        ("DAWG + block strings", dawg_total),
+        ("MPHF no-fp + strings", mphf_total_nofp),
+        ("MPHF 8-bit fp", mphf_total_8),
+        ("LOUDS + strings", louds_total),
+        ("LOUDS rank-index", h3_total),
+        ("FST (optimistic)", fst_low),
+        ("FST (conservative)", fst_high),
+        ("Hybrid 1: Compute+DAWG", h1_total),
+        ("Hybrid 2: MPHF+block", h2_total),
+        ("Hybrid 3: LOUDS+rank", h3_total),
+        ("Hybrid 4: Compute+LOUDS", h4_total),
+        ("Hybrid 5: Compute+MPHF", h5_total),
+        ("Hybrid 6: DAWG+aggr.str", dawg_bytes + bc_total_1k),
+    ]
+
+    approaches.sort(key=lambda x: x[1])
+
+    for name, size in approaches:
+        kb = size / 1024
+        marker = "  ✓" if kb <= 300 else f"  ({kb-300:+.0f} KB over)"
+        print(f"  {name:28s}: {kb:8.1f} KB{marker}")
+
+
+if __name__ == '__main__':
+    main()

tools/dawg_fst_prototype.py

@@ -0,0 +1,722 @@
+#!/usr/bin/env python3
+"""Prototype: DAWG with FST-style edge outputs.
+
+DAWG handles key structure (22K nodes, 87.8% suffix sharing).
+Outputs encoded directly on edges — no separate value index.
+
+When traversing stroke sequence, accumulate output fragments from edges.
+Final node's output = concatenation of all edge outputs along path.
+
+String dedup: outputs reference into block-compressed string table.
+"""
+
+import json
+import struct
+import sys
+import zlib
+import math
+import os
+from collections import Counter, defaultdict
+
+# ─── Steno stroke parsing ───
+
+STENO_KEYS = {
+    '#': 0x00400000,
+    'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004,
+    'P-': 0x00000008, 'W-': 0x00000010, 'H-': 0x00000020,
+    'R-': 0x00000040, 'A-': 0x00000080, 'O-': 0x00000100,
+    '*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800,
+    '-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000,
+    '-B': 0x00008000, '-L': 0x00010000, '-G': 0x00020000,
+    '-T': 0x00040000, '-S': 0x00080000, '-D': 0x00100000,
+    '-Z': 0x00200000,
+}
+
+IMPLICIT_HYPHEN = set('AOEU*')
+
+def parse_stroke(s):
+    result = 0
+    if '#' in s:
+        result |= STENO_KEYS['#']
+        s = s.replace('#', '')
+    has_hyphen = '-' in s
+    s_clean = s.replace('-', '')
+    if not has_hyphen and not any(c in IMPLICIT_HYPHEN for c in s_clean):
+        for c in s_clean:
+            key = c + '-'
+            if key in STENO_KEYS:
+                result |= STENO_KEYS[key]
+        return result
+    past_vowels = False
+    for c in s_clean:
+        if c in 'AO':
+            result |= STENO_KEYS[c + '-']
+            past_vowels = True
+        elif c in 'EU':
+            result |= STENO_KEYS['-' + c]
+            past_vowels = True
+        elif c == '*':
+            result |= STENO_KEYS['*']
+            past_vowels = True
+        elif not past_vowels and (c + '-') in STENO_KEYS:
+            result |= STENO_KEYS[c + '-']
+        elif past_vowels and ('-' + c) in STENO_KEYS:
+            result |= STENO_KEYS['-' + c]
+        elif has_hyphen:
+            if s.index(c) < s.index('-'):
+                result |= STENO_KEYS.get(c + '-', 0)
+            else:
+                result |= STENO_KEYS.get('-' + c, 0)
+        else:
+            if (c + '-') in STENO_KEYS:
+                result |= STENO_KEYS[c + '-']
+    return result
+
+
+# ─── DAWG with outputs ───
+
+class DawgNode:
+    next_id = 0
+    def __init__(self):
+        self.id = DawgNode.next_id
+        DawgNode.next_id += 1
+        self.edges = {}      # stroke_val -> (DawgNode, output_str)
+        self.final = False
+        self.final_output = ""  # remaining output at final state
+
+    def signature(self):
+        """Signature for minimization — includes outputs."""
+        edge_sig = tuple(sorted(
+            (k, child.id, out) for k, (child, out) in self.edges.items()
+        ))
+        return (self.final, self.final_output if self.final else "", edge_sig)
+
+    def __hash__(self):
+        return hash(self.signature())
+
+    def __eq__(self, other):
+        return self.signature() == other.signature()
+
+
+def build_dawg_fst(entries):
+    """
+    Build DAWG with FST-style outputs.
+
+    For each entry (strokes, translation):
+    - Walk trie path for strokes
+    - Attach output to FIRST edge (FST convention: push output left)
+    - At final node, store remaining output
+
+    Then minimize: merge nodes with identical futures (including outputs).
+
+    This is a simplified FST construction — not fully optimal but
+    captures most of the savings.
+    """
+    DawgNode.next_id = 0
+
+    # Phase 1: Build trie with outputs
+    root = DawgNode()
+
+    for strokes, translation in entries:
+        node = root
+        for i, stroke in enumerate(strokes):
+            if stroke not in node.edges:
+                new_node = DawgNode()
+                node.edges[stroke] = (new_node, "")
+                node = new_node
+            else:
+                node = node.edges[stroke][0]
+        node.final = True
+        node.final_output = translation
+
+    # Phase 2: Push outputs to edges (left-push)
+    # For each node, find common prefix of all outputs reachable,
+    # push that prefix to the incoming edge, strip from descendants.
+    # This enables more suffix sharing.
+
+    def push_outputs(node, depth=0):
+        """Push common output prefixes toward the root."""
+        if not node.edges:
+            return
+
+        # First recurse into children
+        for stroke, (child, out) in list(node.edges.items()):
+            push_outputs(child, depth + 1)
+
+        # For each child, collect all outputs reachable from it
+        for stroke, (child, edge_out) in list(node.edges.items()):
+            if child.final and not child.edges:
+                # Leaf: output = edge_out + final_output
+                full_out = edge_out + child.final_output
+                node.edges[stroke] = (child, full_out)
+                child.final_output = ""
+
+    push_outputs(root)
+
+    # Phase 3: Minimize (merge identical subtrees including outputs)
+    minimized = {}
+
+    def minimize_node(node, visited=None):
+        if visited is None:
+            visited = set()
+        if node.id in visited:
+            return node
+        visited.add(node.id)
+
+        # First minimize children
+        for stroke, (child, out) in list(node.edges.items()):
+            minimized_child = minimize_node(child, visited)
+            node.edges[stroke] = (minimized_child, out)
+
+        # Check if we've seen an equivalent node
+        sig = node.signature()
+        if sig in minimized:
+            return minimized[sig]
+        minimized[sig] = node
+        return node
+
+    root = minimize_node(root)
+
+    return root
+
+
+def measure_dawg_fst(root):
+    """Measure the DAWG-FST structure."""
+    nodes = set()
+    edges = 0
+    total_output_bytes = 0
+    output_lengths = []
+    unique_outputs = set()
+
+    def visit(node):
+        nonlocal edges, total_output_bytes
+        if node.id in nodes:
+            return
+        nodes.add(node.id)
+        for stroke, (child, output) in node.edges.items():
+            edges += 1
+            out_bytes = output.encode('utf-8')
+            total_output_bytes += len(out_bytes)
+            output_lengths.append(len(out_bytes))
+            if output:
+                unique_outputs.add(output)
+            visit(child)
+        if node.final and node.final_output:
+            total_output_bytes += len(node.final_output.encode('utf-8'))
+            output_lengths.append(len(node.final_output.encode('utf-8')))
+            unique_outputs.add(node.final_output)
+
+    visit(root)
+    return {
+        'nodes': len(nodes),
+        'edges': edges,
+        'total_output_bytes': total_output_bytes,
+        'unique_outputs': len(unique_outputs),
+        'avg_output_len': sum(output_lengths) / max(len(output_lengths), 1),
+        'output_lengths': output_lengths,
+    }
+
+
+def verify_dawg_fst(root, entries):
+    """Verify lookups return correct translations."""
+    correct = 0
+    wrong = 0
+    missing = 0
+
+    for strokes, expected in entries:
+        node = root
+        output = ""
+        found = True
+        for stroke in strokes:
+            if stroke in node.edges:
+                child, edge_out = node.edges[stroke]
+                output += edge_out
+                node = child
+            else:
+                found = False
+                break
+
+        if found and node.final:
+            output += node.final_output
+            if output == expected:
+                correct += 1
+            else:
+                wrong += 1
+                if wrong <= 5:
+                    print(f"  WRONG: expected '{expected}', got '{output}'")
+        else:
+            missing += 1
+            if missing <= 5:
+                print(f"  MISSING: {strokes} -> '{expected}'")
+
+    return correct, wrong, missing
+
+
+def estimate_binary_size(stats, n_unique_strokes):
+    """Estimate binary encoding size."""
+    n_nodes = stats['nodes']
+    n_edges = stats['edges']
+
+    cbits = max(1, math.ceil(math.log2(max(n_unique_strokes, 2))))
+    abits = max(1, math.ceil(math.log2(max(n_nodes, 2))))
+
+    # Edge encoding: stroke_key + target_node + output_ref
+    # output_ref: index into output string table
+    unique_outputs = stats['unique_outputs']
+    obits = max(1, math.ceil(math.log2(max(unique_outputs + 1, 2))))  # +1 for "no output"
+
+    bits_per_edge = cbits + abits + obits + 1  # +1 for last-edge flag
+    total_edge_bits = n_edges * bits_per_edge
+
+    # Node overhead: 1 bit for is_final, 1 bit for has_final_output
+    node_bits = n_nodes * 2
+
+    # Final outputs: nodes with final_output need an output reference
+    # Approximate: ~50% of final nodes have output
+    final_output_bits = n_nodes * obits // 4  # rough
+
+    structure_bits = total_edge_bits + node_bits + final_output_bits
+    structure_bytes = (structure_bits + 7) // 8
+
+    return {
+        'cbits': cbits,
+        'abits': abits,
+        'obits': obits,
+        'bits_per_edge': bits_per_edge,
+        'structure_bytes': structure_bytes,
+    }
+
+
+def build_output_string_table(root):
+    """Collect all unique output strings and build compressed table."""
+    outputs = set()
+
+    def visit(node, visited=None):
+        if visited is None:
+            visited = set()
+        if node.id in visited:
+            return
+        visited.add(node.id)
+        for stroke, (child, output) in node.edges.items():
+            if output:
+                outputs.add(output)
+            visit(child, visited)
+        if node.final and node.final_output:
+            outputs.add(node.final_output)
+
+    visit(root)
+
+    sorted_outputs = sorted(outputs)
+
+    # Raw
+    raw = b'\x00'.join(o.encode('utf-8') for o in sorted_outputs)
+    raw_size = len(raw)
+
+    # Block compressed
+    block_size = 4096
+    blocks = []
+    for i in range(0, len(raw), block_size):
+        block = raw[i:i+block_size]
+        blocks.append(zlib.compress(block, 9))
+    compressed_size = sum(len(b) for b in blocks)
+    index_size = len(blocks) * 4
+
+    # Full zlib
+    full_zlib_size = len(zlib.compress(raw, 9))
+
+    return {
+        'unique_count': len(sorted_outputs),
+        'raw_size': raw_size,
+        'block_compressed': compressed_size + index_size,
+        'full_zlib': full_zlib_size,
+    }
+
+
+# ─── Alternative: DAWG keys + implicit index + compressed values ───
+
+def build_dawg_implicit_index(entries):
+    """
+    Standard DAWG (no outputs on edges) but with implicit indexing.
+
+    DAWG traversal counts reachable final nodes → gives entry index.
+    Entry index maps to value via simple array lookup.
+
+    Values stored as: entry_index → string_table_offset
+    String table block-compressed.
+
+    Key difference from benchmark.py: here we build actual DAWG
+    and measure REAL node count, then compute skip-count based size.
+    """
+    DawgNode.next_id = 0
+    root = DawgNode()
+    unchecked = []
+    minimized = {}
+    prev_strokes = []
+
+    def minimize(down_to):
+        for i in range(len(unchecked) - 1, down_to - 1, -1):
+            parent, stroke, child = unchecked[i]
+            sig = (child.final, tuple(sorted((k, v.id) for k, (v, _) in child.edges.items())))
+            if sig in minimized:
+                existing = minimized[sig]
+                parent.edges[stroke] = (existing, "")
+            else:
+                minimized[sig] = child
+            unchecked.pop()
+
+    for strokes, translation in entries:
+        common = 0
+        for i in range(min(len(strokes), len(prev_strokes))):
+            if strokes[i] != prev_strokes[i]:
+                break
+            common += 1
+        else:
+            common = min(len(strokes), len(prev_strokes))
+
+        minimize(common)
+
+        if unchecked:
+            node = unchecked[-1][2]
+        else:
+            node = root
+
+        for stroke in strokes[common:]:
+            new_node = DawgNode()
+            node.edges[stroke] = (new_node, "")
+            unchecked.append((node, stroke, new_node))
+            node = new_node
+
+        node.final = True
+        prev_strokes = strokes
+
+    minimize(0)
+
+    # Count reachable finals for skip-count indexing
+    def count_finals(node, cache=None):
+        if cache is None:
+            cache = {}
+        if node.id in cache:
+            return cache[node.id]
+        c = 1 if node.final else 0
+        for stroke in sorted(node.edges.keys()):
+            child, _ = node.edges[stroke]
+            c += count_finals(child, cache)
+        cache[node.id] = c
+        return c
+
+    count_finals(root)
+
+    return root
+
+
+def verify_implicit_index(root, entries):
+    """Verify skip-count indexing gives correct sequential indices."""
+
+    def lookup_index(node, strokes):
+        """Return the skip-count index for a stroke sequence."""
+        idx = 0
+        for stroke in strokes:
+            # Count finals of all children with stroke < target
+            for s in sorted(node.edges.keys()):
+                if s == stroke:
+                    child, _ = node.edges[s]
+                    if child.final:
+                        # This child's final state comes before its children
+                        pass
+                    node = child
+                    if node.final:
+                        idx += 1  # count this final state
+                    break
+                else:
+                    child, _ = node.edges[s]
+                    idx += count_subtree_finals(child)
+            else:
+                return -1
+        return idx - 1 if node.final else -1
+
+    def count_subtree_finals(node, cache={}):
+        if node.id in cache:
+            return cache[node.id]
+        c = 1 if node.final else 0
+        for s in sorted(node.edges.keys()):
+            child, _ = node.edges[s]
+            c += count_subtree_finals(child, cache)
+        cache[node.id] = c
+        return c
+
+    # Verify first 100 entries get sequential indices
+    correct = 0
+    for expected_idx, (strokes, translation) in enumerate(entries[:100]):
+        got_idx = lookup_index(root, strokes)
+        if got_idx == expected_idx:
+            correct += 1
+
+    return correct
+
+
+# ─── Main ───
+
+def main():
+    dict_path = '/tmp/plover-main.json'
+    if not os.path.exists(dict_path):
+        print("ERROR: Download Plover dict first:")
+        print("  curl -sL 'https://raw.githubusercontent.com/openstenoproject/plover/main/plover/assets/main.json' -o /tmp/plover-main.json")
+        sys.exit(1)
+
+    with open(dict_path) as f:
+        raw_dict = json.load(f)
+
+    # Parse and sort
+    parsed = []
+    unique_stroke_vals = set()
+    for stroke_str in sorted(raw_dict.keys()):
+        strokes = tuple(parse_stroke(s) for s in stroke_str.split('/'))
+        for s in strokes:
+            unique_stroke_vals.add(s)
+        parsed.append((strokes, raw_dict[stroke_str]))
+    parsed.sort(key=lambda x: x[0])
+
+    n_entries = len(parsed)
+    n_unique_strokes = len(unique_stroke_vals)
+    translations = [v for _, v in parsed]
+
+    print(f"Entries: {n_entries}, Unique strokes: {n_unique_strokes}")
+    print()
+
+    # ─── Approach A: DAWG-FST (outputs on edges) ───
+    print("=" * 60)
+    print("APPROACH A: DAWG-FST (outputs on edges)")
+    print("=" * 60)
+
+    print("Building DAWG-FST...")
+    root_fst = build_dawg_fst(parsed)
+
+    stats = measure_dawg_fst(root_fst)
+    print(f"  Nodes: {stats['nodes']}")
+    print(f"  Edges: {stats['edges']}")
+    print(f"  Unique output strings: {stats['unique_outputs']}")
+    print(f"  Total output bytes (on edges): {stats['total_output_bytes']}")
+    print(f"  Avg output length: {stats['avg_output_len']:.1f} bytes")
+
+    # Verify correctness
+    print("  Verifying lookups...")
+    correct, wrong, missing = verify_dawg_fst(root_fst, parsed[:1000])
+    print(f"  Verification (first 1000): {correct} correct, {wrong} wrong, {missing} missing")
+
+    # Output string table
+    str_table = build_output_string_table(root_fst)
+    print(f"  Output string table:")
+    print(f"    Unique strings: {str_table['unique_count']}")
+    print(f"    Raw: {str_table['raw_size']/1024:.1f} KB")
+    print(f"    Block-compressed: {str_table['block_compressed']/1024:.1f} KB")
+    print(f"    Full zlib: {str_table['full_zlib']/1024:.1f} KB")
+
+    # Binary size estimate
+    bin_est = estimate_binary_size(stats, n_unique_strokes)
+    print(f"  Binary encoding:")
+    print(f"    cbits={bin_est['cbits']} abits={bin_est['abits']} obits={bin_est['obits']}")
+    print(f"    Bits/edge: {bin_est['bits_per_edge']}")
+    print(f"    Structure: {bin_est['structure_bytes']/1024:.1f} KB")
+
+    total_a = bin_est['structure_bytes'] + str_table['block_compressed']
+    print(f"  TOTAL: {total_a/1024:.1f} KB")
+    print()
+
+    # ─── Approach B: DAWG + implicit skip-count index ───
+    print("=" * 60)
+    print("APPROACH B: DAWG + skip-count index (no value array)")
+    print("=" * 60)
+
+    print("Building standard DAWG...")
+    root_std = build_dawg_implicit_index(parsed)
+
+    stats_std = measure_dawg_fst(root_std)  # reuse measurement fn
+    print(f"  Nodes: {stats_std['nodes']}")
+    print(f"  Edges: {stats_std['edges']}")
+
+    # DAWG structure: edges need stroke + target + skip_count
+    cbits = max(1, math.ceil(math.log2(max(n_unique_strokes, 2))))
+    abits = max(1, math.ceil(math.log2(max(stats_std['nodes'], 2))))
+    skipbits = max(1, math.ceil(math.log2(max(n_entries, 2))))
+
+    # Compact: last-edge flag saves storing child count
+    bits_per_edge = cbits + abits + skipbits + 1  # stroke + target + skip + last_edge
+    structure_bits = stats_std['edges'] * bits_per_edge + stats_std['nodes'] * 1  # is_final per node
+    structure_bytes = (structure_bits + 7) // 8
+
+    print(f"  cbits={cbits} abits={abits} skipbits={skipbits}")
+    print(f"  Bits/edge: {bits_per_edge}")
+    print(f"  Structure: {structure_bytes/1024:.1f} KB")
+
+    # Values: skip-count gives index → look up in ordered value array
+    # Value array: translations in DAWG traversal order
+    # Need: string table + offset array (index → string table position)
+    unique_trans = sorted(set(translations))
+    raw_strings = b'\x00'.join(t.encode('utf-8') for t in unique_trans)
+
+    # Dedup: entry → unique_string_id
+    trans_to_id = {t: i for i, t in enumerate(unique_trans)}
+    dedup_array = [trans_to_id[t] for t in translations]
+    dedup_bits = math.ceil(math.log2(len(unique_trans)))
+    dedup_bytes = (n_entries * dedup_bits + 7) // 8
+
+    # String table block-compressed
+    blocks = []
+    for i in range(0, len(raw_strings), 4096):
+        blocks.append(zlib.compress(raw_strings[i:i+4096], 9))
+    str_compressed = sum(len(b) for b in blocks) + len(blocks) * 4
+
+    total_b = structure_bytes + dedup_bytes + str_compressed
+    print(f"  Value dedup array: {dedup_bytes/1024:.1f} KB ({dedup_bits} bits/entry)")
+    print(f"  String table: {str_compressed/1024:.1f} KB")
+    print(f"  TOTAL: {total_b/1024:.1f} KB")
+    print()
+
+    # ─── Approach C: DAWG + skip-count + NO dedup array ───
+    # Instead of dedup array, store string offset directly in DAWG final nodes
+    print("=" * 60)
+    print("APPROACH C: DAWG + skip-count + direct string refs")
+    print("=" * 60)
+
+    # Each final node stores a string table offset
+    n_final = sum(1 for _ in range(1))  # need to count
+    visited_c = set()
+    n_final_c = 0
+    def count_final(node):
+        nonlocal n_final_c
+        if node.id in visited_c:
+            return
+        visited_c.add(node.id)
+        if node.final:
+            n_final_c += 1
+        for s, (child, _) in node.edges.items():
+            count_final(child)
+    count_final(root_std)
+
+    # But wait — with DAWG suffix sharing, multiple entries share final nodes
+    # A shared final node can only store ONE string offset
+    # This breaks dedup... unless we use the skip-count to disambiguate
+    # Skip-count already gives unique index → use that as index into value array
+    # So we STILL need the value array
+
+    # Alternative: don't share final nodes (partial DAWG — share internal only)
+    # Then each final node = unique entry = unique string ref
+    print(f"  Final nodes (shared): {n_final_c}")
+    print(f"  Total entries: {n_entries}")
+    print(f"  Final nodes can't store unique refs with suffix sharing")
+    print(f"  → Must use skip-count index + value array (same as Approach B)")
+    print()
+
+    # ─── Approach D: Partial DAWG (share internal only) + direct refs ───
+    print("=" * 60)
+    print("APPROACH D: Partial DAWG (no suffix sharing at finals)")
+    print("=" * 60)
+
+    # Don't merge final nodes → each has unique string ref
+    # Merge only internal nodes
+    # Trade: more nodes but no value array needed
+
+    # In the standard DAWG we had 22K nodes.
+    # Without suffix sharing at finals, estimate:
+    # 147K entries = 147K unique final nodes + shared internal nodes
+    # Internal nodes from DAWG: ~22K - final_shared ≈ much more nodes
+    # This blows up the structure. Not good.
+    print(f"  Would need ~{n_entries} final nodes (no sharing)")
+    print(f"  Structure would be larger than value array savings")
+    print(f"  → Not viable")
+    print()
+
+    # ─── Approach E: DAWG + varint value array ───
+    print("=" * 60)
+    print("APPROACH E: DAWG + varint-compressed value array")
+    print("=" * 60)
+
+    # Instead of fixed-width dedup_bits per entry, use varint
+    # Most translations are common (dedup IDs are small for frequent ones)
+    # Sort unique translations by frequency → frequent = small ID → small varint
+
+    trans_freq = Counter(translations)
+    sorted_by_freq = sorted(set(translations), key=lambda t: -trans_freq[t])
+    freq_to_id = {t: i for i, t in enumerate(sorted_by_freq)}
+
+    # Varint encode: 7 bits per byte, high bit = continuation
+    def varint_size(n):
+        if n < 128: return 1
+        if n < 16384: return 2
+        if n < 2097152: return 3
+        return 4
+
+    varint_total = sum(varint_size(freq_to_id[t]) for t in translations)
+
+    print(f"  Fixed-width value array: {dedup_bytes/1024:.1f} KB")
+    print(f"  Varint value array: {varint_total/1024:.1f} KB")
+    print(f"  Savings: {(dedup_bytes - varint_total)/1024:.1f} KB")
+
+    total_e = structure_bytes + varint_total + str_compressed
+    print(f"  TOTAL: {total_e/1024:.1f} KB")
+    print()
+
+    # ─── Approach F: Hybrid — DAWG keys + Huffman-coded values ───
+    print("=" * 60)
+    print("APPROACH F: DAWG + Huffman-coded value IDs")
+    print("=" * 60)
+
+    # Huffman code the dedup IDs based on frequency
+    # Theoretical minimum: entropy
+    total_entries = len(translations)
+    entropy_bits = 0
+    for t, count in trans_freq.items():
+        p = count / total_entries
+        entropy_bits -= count * math.log2(p)
+    entropy_bytes = int(entropy_bits / 8)
+
+    print(f"  Entropy of value mapping: {entropy_bytes/1024:.1f} KB")
+    print(f"  (theoretical minimum for value array)")
+
+    total_f = structure_bytes + entropy_bytes + str_compressed
+    print(f"  DAWG structure: {structure_bytes/1024:.1f} KB")
+    print(f"  Huffman values: {entropy_bytes/1024:.1f} KB")
+    print(f"  String table: {str_compressed/1024:.1f} KB")
+    print(f"  TOTAL: {total_f/1024:.1f} KB")
+    print()
+
+    # ─── Summary ───
+    print("=" * 60)
+    print("SUMMARY")
+    print("=" * 60)
+    print(f"  Target: 300 KB")
+    print()
+
+    results = [
+        ("A: DAWG-FST (edge outputs)", total_a),
+        ("B: DAWG + skip + dedup array", total_b),
+        ("E: DAWG + varint values", total_e),
+        ("F: DAWG + Huffman values", total_f),
+    ]
+    results.sort(key=lambda x: x[1])
+
+    for name, size in results:
+        kb = size / 1024
+        marker = "  ✓ FITS!" if kb <= 300 else f"  ({kb-300:+.0f} KB over)"
+        print(f"  {name:40s}: {kb:8.1f} KB{marker}")
+
+    # Breakdown of best approach
+    print()
+    best_name, best_size = results[0]
+    print(f"  Best: {best_name}")
+    print(f"  Breakdown:")
+    if "FST" in best_name:
+        print(f"    DAWG-FST structure: {bin_est['structure_bytes']/1024:.1f} KB")
+        print(f"    Output string table: {str_table['block_compressed']/1024:.1f} KB")
+    elif "Huffman" in best_name:
+        print(f"    DAWG structure: {structure_bytes/1024:.1f} KB")
+        print(f"    Huffman value IDs: {entropy_bytes/1024:.1f} KB")
+        print(f"    String table: {str_compressed/1024:.1f} KB")
+    elif "varint" in best_name:
+        print(f"    DAWG structure: {structure_bytes/1024:.1f} KB")
+        print(f"    Varint value array: {varint_total/1024:.1f} KB")
+        print(f"    String table: {str_compressed/1024:.1f} KB")
+
+
+if __name__ == '__main__':
+    main()