Initial project: benchmarks, status doc, gitignore
This commit is contained in:
commit
234bc8d731
4 changed files with 2009 additions and 0 deletions
0
.gitignore
vendored
Normal file
0
.gitignore
vendored
Normal file
600
tools/alt_approaches.py
Normal file
600
tools/alt_approaches.py
Normal file
|
|
@ -0,0 +1,600 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Alternative compression approaches beyond DAWG/trie.
|
||||
|
||||
1. Compressed page table (sorted entries, zlib pages, binary search)
|
||||
2. Rule-based reduction (phonetic rules + exception table)
|
||||
3. MPHF + compressed blob (no random access, decompress per-lookup)
|
||||
4. Two-level hash (first stroke → bucket → scan)
|
||||
5. Full compressed blob with LRU page cache
|
||||
6. Hybrid combos
|
||||
"""
|
||||
|
||||
import json
|
||||
import struct
|
||||
import sys
|
||||
import zlib
|
||||
import math
|
||||
import os
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
STENO_KEYS = {
|
||||
'#': 0x00400000,
|
||||
'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004,
|
||||
'P-': 0x00000008, 'W-': 0x00000010, 'H-': 0x00000020,
|
||||
'R-': 0x00000040, 'A-': 0x00000080, 'O-': 0x00000100,
|
||||
'*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800,
|
||||
'-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000,
|
||||
'-B': 0x00008000, '-L': 0x00010000, '-G': 0x00020000,
|
||||
'-T': 0x00040000, '-S': 0x00080000, '-D': 0x00100000,
|
||||
'-Z': 0x00200000,
|
||||
}
|
||||
IMPLICIT_HYPHEN = set('AOEU*')
|
||||
|
||||
def parse_stroke(s):
|
||||
result = 0
|
||||
if '#' in s:
|
||||
result |= STENO_KEYS['#']
|
||||
s = s.replace('#', '')
|
||||
has_hyphen = '-' in s
|
||||
s_clean = s.replace('-', '')
|
||||
if not has_hyphen and not any(c in IMPLICIT_HYPHEN for c in s_clean):
|
||||
for c in s_clean:
|
||||
key = c + '-'
|
||||
if key in STENO_KEYS:
|
||||
result |= STENO_KEYS[key]
|
||||
return result
|
||||
past_vowels = False
|
||||
for c in s_clean:
|
||||
if c in 'AO':
|
||||
result |= STENO_KEYS[c + '-']
|
||||
past_vowels = True
|
||||
elif c in 'EU':
|
||||
result |= STENO_KEYS['-' + c]
|
||||
past_vowels = True
|
||||
elif c == '*':
|
||||
result |= STENO_KEYS['*']
|
||||
past_vowels = True
|
||||
elif not past_vowels and (c + '-') in STENO_KEYS:
|
||||
result |= STENO_KEYS[c + '-']
|
||||
elif past_vowels and ('-' + c) in STENO_KEYS:
|
||||
result |= STENO_KEYS['-' + c]
|
||||
elif has_hyphen:
|
||||
if s.index(c) < s.index('-'):
|
||||
result |= STENO_KEYS.get(c + '-', 0)
|
||||
else:
|
||||
result |= STENO_KEYS.get('-' + c, 0)
|
||||
else:
|
||||
if (c + '-') in STENO_KEYS:
|
||||
result |= STENO_KEYS[c + '-']
|
||||
return result
|
||||
|
||||
|
||||
def stroke_to_bytes(stroke_val):
|
||||
"""Encode stroke as 3 bytes (23 bits used)."""
|
||||
return struct.pack('<I', stroke_val)[:3]
|
||||
|
||||
|
||||
# ─── Approach 1: Compressed Page Table ───
|
||||
|
||||
def test_compressed_pages(entries, page_sizes=[64, 128, 256, 512]):
|
||||
"""
|
||||
Sort entries, group into pages, delta-encode + zlib each page.
|
||||
Binary search on page index for lookup.
|
||||
"""
|
||||
print("=" * 60)
|
||||
print("APPROACH 1: COMPRESSED PAGE TABLE")
|
||||
print("=" * 60)
|
||||
|
||||
# Build string table (deduped, sorted)
|
||||
all_translations = sorted(set(v for _, v in entries))
|
||||
trans_to_id = {t: i for i, t in enumerate(all_translations)}
|
||||
|
||||
raw_strings = b'\x00'.join(t.encode('utf-8') for t in all_translations)
|
||||
str_compressed = len(zlib.compress(raw_strings, 9))
|
||||
|
||||
# Block-compressed strings for random access
|
||||
str_blocks = []
|
||||
for i in range(0, len(raw_strings), 4096):
|
||||
str_blocks.append(zlib.compress(raw_strings[i:i+4096], 9))
|
||||
str_block_total = sum(len(b) for b in str_blocks) + len(str_blocks) * 4
|
||||
|
||||
print(f" String table (block 4KB): {str_block_total/1024:.1f} KB")
|
||||
print(f" String table (full zlib): {str_compressed/1024:.1f} KB")
|
||||
print()
|
||||
|
||||
for page_size in page_sizes:
|
||||
n_pages = math.ceil(len(entries) / page_size)
|
||||
|
||||
# Serialize each page: delta-coded stroke sequences + translation IDs
|
||||
page_data = []
|
||||
page_index = [] # (first_key, compressed_offset)
|
||||
|
||||
total_compressed = 0
|
||||
|
||||
for p in range(n_pages):
|
||||
start = p * page_size
|
||||
end = min(start + page_size, len(entries))
|
||||
page_entries = entries[start:end]
|
||||
|
||||
# First key for index
|
||||
first_key = page_entries[0][0]
|
||||
|
||||
# Serialize page
|
||||
buf = bytearray()
|
||||
prev_strokes = ()
|
||||
|
||||
for strokes, translation in page_entries:
|
||||
tid = trans_to_id[translation]
|
||||
|
||||
# Delta from previous: shared prefix length + new strokes + tid
|
||||
shared = 0
|
||||
for i in range(min(len(strokes), len(prev_strokes))):
|
||||
if strokes[i] == prev_strokes[i]:
|
||||
shared += 1
|
||||
else:
|
||||
break
|
||||
|
||||
# Encode: shared_len(1) + n_new(1) + new_strokes(3 each) + tid(2)
|
||||
n_new = len(strokes) - shared
|
||||
buf.append(shared)
|
||||
buf.append(n_new)
|
||||
for s in strokes[shared:]:
|
||||
buf.extend(stroke_to_bytes(s))
|
||||
buf.extend(struct.pack('<H', tid & 0xFFFF))
|
||||
|
||||
prev_strokes = strokes
|
||||
|
||||
raw_page = bytes(buf)
|
||||
compressed_page = zlib.compress(raw_page, 9)
|
||||
|
||||
page_index.append((first_key, total_compressed, len(compressed_page)))
|
||||
total_compressed += len(compressed_page)
|
||||
page_data.append(compressed_page)
|
||||
|
||||
# Page index size: per page = first_key (variable) + offset(3) + size(2)
|
||||
avg_key_len = sum(len(k) * 3 for k, _, _ in page_index) / len(page_index)
|
||||
index_entry_size = avg_key_len + 5
|
||||
index_total = int(n_pages * index_entry_size)
|
||||
|
||||
# Simpler: first_key_hash(4) + offset(3) + size(2) = 9 bytes per page
|
||||
index_simple = n_pages * 9
|
||||
|
||||
total = total_compressed + index_simple + str_block_total
|
||||
total_fullzlib = total_compressed + index_simple + str_compressed
|
||||
|
||||
print(f" Page size={page_size:4d}: {n_pages:4d} pages, "
|
||||
f"pages={total_compressed/1024:.1f}KB "
|
||||
f"idx={index_simple/1024:.1f}KB "
|
||||
f"str={str_block_total/1024:.1f}KB "
|
||||
f"TOTAL={total/1024:.1f}KB "
|
||||
f"(w/full zlib str: {total_fullzlib/1024:.1f}KB)")
|
||||
|
||||
print()
|
||||
return total_compressed, index_simple, str_block_total
|
||||
|
||||
|
||||
# ─── Approach 2: Two-Level Hash Table ───
|
||||
|
||||
def test_two_level_hash(entries):
|
||||
"""
|
||||
Level 1: hash(first_stroke) → bucket
|
||||
Level 2: within bucket, linear scan of entries
|
||||
Each bucket independently compressed.
|
||||
"""
|
||||
print("=" * 60)
|
||||
print("APPROACH 2: TWO-LEVEL HASH TABLE")
|
||||
print("=" * 60)
|
||||
|
||||
# Group by first stroke
|
||||
buckets = defaultdict(list)
|
||||
for strokes, translation in entries:
|
||||
buckets[strokes[0]].append((strokes, translation))
|
||||
|
||||
print(f" Unique first strokes (buckets): {len(buckets)}")
|
||||
|
||||
bucket_sizes = [len(v) for v in buckets.values()]
|
||||
print(f" Bucket size: min={min(bucket_sizes)} max={max(bucket_sizes)} "
|
||||
f"avg={sum(bucket_sizes)/len(bucket_sizes):.1f} "
|
||||
f"median={sorted(bucket_sizes)[len(bucket_sizes)//2]}")
|
||||
|
||||
# Build string table
|
||||
all_translations = sorted(set(v for _, v in entries))
|
||||
trans_to_id = {t: i for i, t in enumerate(all_translations)}
|
||||
raw_strings = b'\x00'.join(t.encode('utf-8') for t in all_translations)
|
||||
str_blocks = []
|
||||
for i in range(0, len(raw_strings), 4096):
|
||||
str_blocks.append(zlib.compress(raw_strings[i:i+4096], 9))
|
||||
str_total = sum(len(b) for b in str_blocks) + len(str_blocks) * 4
|
||||
|
||||
# Compress each bucket
|
||||
total_bucket_compressed = 0
|
||||
for first_stroke, bucket_entries in buckets.items():
|
||||
buf = bytearray()
|
||||
for strokes, translation in sorted(bucket_entries):
|
||||
tid = trans_to_id[translation]
|
||||
# remaining strokes after first
|
||||
remaining = strokes[1:]
|
||||
buf.append(len(remaining))
|
||||
for s in remaining:
|
||||
buf.extend(stroke_to_bytes(s))
|
||||
buf.extend(struct.pack('<H', tid & 0xFFFF))
|
||||
|
||||
compressed = zlib.compress(bytes(buf), 9)
|
||||
total_bucket_compressed += len(compressed)
|
||||
|
||||
# Bucket index: first_stroke(3) + offset(3) + size(2) = 8 bytes each
|
||||
bucket_index = len(buckets) * 8
|
||||
|
||||
total = total_bucket_compressed + bucket_index + str_total
|
||||
print(f" Buckets compressed: {total_bucket_compressed/1024:.1f} KB")
|
||||
print(f" Bucket index: {bucket_index/1024:.1f} KB")
|
||||
print(f" String table: {str_total/1024:.1f} KB")
|
||||
print(f" TOTAL: {total/1024:.1f} KB")
|
||||
|
||||
# With full zlib strings
|
||||
str_zlib = len(zlib.compress(raw_strings, 9))
|
||||
total_zlib = total_bucket_compressed + bucket_index + str_zlib
|
||||
print(f" TOTAL (full zlib str): {total_zlib/1024:.1f} KB")
|
||||
print()
|
||||
return total
|
||||
|
||||
|
||||
# ─── Approach 3: Full zlib blob with page cache ───
|
||||
|
||||
def test_full_compressed(entries):
|
||||
"""
|
||||
Store everything as one zlib blob. At runtime, decompress
|
||||
needed sections into RAM page cache. LRU eviction.
|
||||
"""
|
||||
print("=" * 60)
|
||||
print("APPROACH 3: FULL COMPRESSED BLOB + PAGE CACHE")
|
||||
print("=" * 60)
|
||||
|
||||
all_translations = sorted(set(v for _, v in entries))
|
||||
trans_to_id = {t: i for i, t in enumerate(all_translations)}
|
||||
|
||||
# Serialize all entries sorted
|
||||
buf = bytearray()
|
||||
prev_strokes = ()
|
||||
for strokes, translation in entries:
|
||||
tid = trans_to_id[translation]
|
||||
shared = 0
|
||||
for i in range(min(len(strokes), len(prev_strokes))):
|
||||
if strokes[i] == prev_strokes[i]:
|
||||
shared += 1
|
||||
else:
|
||||
break
|
||||
n_new = len(strokes) - shared
|
||||
buf.append(shared)
|
||||
buf.append(n_new)
|
||||
for s in strokes[shared:]:
|
||||
buf.extend(stroke_to_bytes(s))
|
||||
buf.extend(struct.pack('<H', tid & 0xFFFF))
|
||||
prev_strokes = strokes
|
||||
|
||||
raw_entries = bytes(buf)
|
||||
|
||||
# String table
|
||||
raw_strings = b'\x00'.join(t.encode('utf-8') for t in all_translations)
|
||||
|
||||
# Full zlib
|
||||
entries_zlib = len(zlib.compress(raw_entries, 9))
|
||||
strings_zlib = len(zlib.compress(raw_strings, 9))
|
||||
|
||||
# zstd-like (try different zlib levels and strategies)
|
||||
entries_zlib_best = len(zlib.compress(raw_entries, 9))
|
||||
|
||||
print(f" Raw entries: {len(raw_entries)/1024:.1f} KB")
|
||||
print(f" Raw strings: {len(raw_strings)/1024:.1f} KB")
|
||||
print(f" Entries zlib: {entries_zlib/1024:.1f} KB")
|
||||
print(f" Strings zlib: {strings_zlib/1024:.1f} KB")
|
||||
print(f" TOTAL (full zlib): {(entries_zlib + strings_zlib)/1024:.1f} KB")
|
||||
print()
|
||||
print(f" Runtime: decompress ~4KB page per lookup (~1-5ms)")
|
||||
print(f" RAM cache: 8 pages × 4KB = 32KB for hot entries")
|
||||
print()
|
||||
|
||||
# Sectored version: split into 4KB sectors, independently compressed
|
||||
# Allows decompressing only the needed sector
|
||||
sector_size = 4096
|
||||
sectors = []
|
||||
for i in range(0, len(raw_entries), sector_size):
|
||||
sector = raw_entries[i:i+sector_size]
|
||||
sectors.append(zlib.compress(sector, 9))
|
||||
|
||||
sectors_total = sum(len(s) for s in sectors)
|
||||
sector_index = len(sectors) * 8 # offset(4) + first_entry_stroke_hash(4)
|
||||
|
||||
str_sectors = []
|
||||
for i in range(0, len(raw_strings), sector_size):
|
||||
str_sectors.append(zlib.compress(raw_strings[i:i+sector_size], 9))
|
||||
str_sectors_total = sum(len(s) for s in str_sectors)
|
||||
str_sector_index = len(str_sectors) * 4
|
||||
|
||||
sectored_total = sectors_total + sector_index + str_sectors_total + str_sector_index
|
||||
|
||||
print(f" Sectored (4KB, random access):")
|
||||
print(f" Entry sectors: {sectors_total/1024:.1f} KB ({len(sectors)} sectors)")
|
||||
print(f" String sectors: {str_sectors_total/1024:.1f} KB ({len(str_sectors)} sectors)")
|
||||
print(f" Indices: {(sector_index + str_sector_index)/1024:.1f} KB")
|
||||
print(f" TOTAL: {sectored_total/1024:.1f} KB")
|
||||
print()
|
||||
|
||||
return entries_zlib + strings_zlib, sectored_total
|
||||
|
||||
|
||||
# ─── Approach 4: Phonetic rule analysis ───
|
||||
|
||||
def test_rule_reduction(raw_dict):
|
||||
"""
|
||||
Analyze how many entries could be generated by phonetic rules.
|
||||
Steno maps sounds → letters. Regular words follow patterns.
|
||||
"""
|
||||
print("=" * 60)
|
||||
print("APPROACH 4: PHONETIC RULE ANALYSIS")
|
||||
print("=" * 60)
|
||||
|
||||
# Steno phonetic mappings (left hand → initial consonants)
|
||||
LEFT_MAP = {
|
||||
0x00000001: 's', # S
|
||||
0x00000002: 't', # T
|
||||
0x00000004: 'k', # K
|
||||
0x00000002|0x00000004: 'c', # TK → d → but also 'c' context dependent
|
||||
0x00000008: 'p', # P
|
||||
0x00000010: 'w', # W
|
||||
0x00000020: 'h', # H
|
||||
0x00000040: 'r', # R
|
||||
}
|
||||
|
||||
# Count single-stroke entries where output is a simple word
|
||||
single_stroke = {k: v for k, v in raw_dict.items() if '/' not in k}
|
||||
multi_stroke = {k: v for k, v in raw_dict.items() if '/' in k}
|
||||
|
||||
# Categorize translations
|
||||
simple_words = 0 # single word, lowercase, no formatting
|
||||
formatted = 0 # contains { } formatting
|
||||
phrases = 0 # multiple words
|
||||
other = 0
|
||||
|
||||
for translation in raw_dict.values():
|
||||
if '{' in translation:
|
||||
formatted += 1
|
||||
elif ' ' in translation.strip():
|
||||
phrases += 1
|
||||
elif translation.strip().replace("'", "").replace("-", "").isalpha():
|
||||
simple_words += 1
|
||||
else:
|
||||
other += 1
|
||||
|
||||
print(f" Translation categories:")
|
||||
print(f" Simple words: {simple_words} ({simple_words/len(raw_dict)*100:.1f}%)")
|
||||
print(f" Phrases: {phrases} ({phrases/len(raw_dict)*100:.1f}%)")
|
||||
print(f" Formatted: {formatted} ({formatted/len(raw_dict)*100:.1f}%)")
|
||||
print(f" Other: {other} ({other/len(raw_dict)*100:.1f}%)")
|
||||
print()
|
||||
|
||||
# Suffix/prefix analysis
|
||||
# Many multi-stroke entries = base word + suffix stroke
|
||||
# If we store base words + suffix rules, we eliminate many entries
|
||||
|
||||
# Check: how many multi-stroke translations are base_word + common_suffix?
|
||||
single_translations = set(raw_dict[k] for k in single_stroke)
|
||||
|
||||
common_suffixes = ['ing', 'ed', 'er', 'est', 'ly', 'ment', 'ness', 'tion',
|
||||
'sion', 'able', 'ible', 'ful', 'less', 'ous', 'ive',
|
||||
'al', 'ial', 's', 'es', "'s", 'ry', 'ary', 'ity',
|
||||
'ize', 'ise', 'en', 'ence', 'ance', 'ent', 'ant',
|
||||
'ion', 'or', 'ist', 'ism', 'ical', 'ically']
|
||||
|
||||
derivable = 0
|
||||
derivable_by_suffix = Counter()
|
||||
|
||||
for k, v in multi_stroke.items():
|
||||
v_clean = v.strip().lower()
|
||||
for suffix in common_suffixes:
|
||||
if v_clean.endswith(suffix):
|
||||
base = v_clean[:-len(suffix)]
|
||||
# Check variants of base in single-stroke dict
|
||||
if base in single_translations or (base + 'e') in single_translations:
|
||||
derivable += 1
|
||||
derivable_by_suffix[suffix] += 1
|
||||
break
|
||||
|
||||
print(f" Multi-stroke entries derivable from single + suffix:")
|
||||
print(f" Derivable: {derivable} of {len(multi_stroke)} ({derivable/len(multi_stroke)*100:.1f}%)")
|
||||
for suffix, count in derivable_by_suffix.most_common(10):
|
||||
print(f" -{suffix}: {count}")
|
||||
|
||||
remaining = len(raw_dict) - derivable
|
||||
print()
|
||||
print(f" If suffix rules eliminate {derivable} entries:")
|
||||
print(f" Remaining: {remaining} entries")
|
||||
print(f" Reduction: {derivable/len(raw_dict)*100:.1f}%")
|
||||
print()
|
||||
|
||||
# Word frequency: many entries are for rare words
|
||||
# English has ~3000 core words covering 95% of text
|
||||
# How many Plover entries map to these core words?
|
||||
|
||||
# Rough: count unique simple output words
|
||||
word_freq = Counter()
|
||||
for v in raw_dict.values():
|
||||
v_clean = v.strip().lower()
|
||||
if v_clean.isalpha():
|
||||
word_freq[v_clean] += 1
|
||||
|
||||
# Top N words cover what % of entries?
|
||||
total_word_entries = sum(word_freq.values())
|
||||
cumulative = 0
|
||||
for n, (word, count) in enumerate(word_freq.most_common()):
|
||||
cumulative += count
|
||||
if n + 1 in (1000, 3000, 5000, 10000, 20000):
|
||||
print(f" Top {n+1:5d} words cover {cumulative/total_word_entries*100:.1f}% "
|
||||
f"of word entries ({cumulative}/{total_word_entries})")
|
||||
|
||||
print()
|
||||
return remaining
|
||||
|
||||
|
||||
# ─── Approach 5: Multi-strategy hybrid ───
|
||||
|
||||
def test_hybrid(entries, raw_dict):
|
||||
"""
|
||||
Best of all approaches combined:
|
||||
1. Phonetic rules for regular derivations (suffix combos)
|
||||
2. Two-level hash for remaining entries
|
||||
3. Block-compressed strings
|
||||
4. LRU cache for hot pages
|
||||
"""
|
||||
print("=" * 60)
|
||||
print("APPROACH 5: MULTI-STRATEGY HYBRID")
|
||||
print("=" * 60)
|
||||
|
||||
# Strategy 1: Suffix rules (from approach 4)
|
||||
single_stroke = {k: v for k, v in raw_dict.items() if '/' not in k}
|
||||
multi_stroke = {k: v for k, v in raw_dict.items() if '/' in k}
|
||||
single_translations = set(raw_dict[k] for k in single_stroke)
|
||||
|
||||
common_suffixes = ['ing', 'ed', 'er', 'est', 'ly', 'ment', 'ness', 'tion',
|
||||
'sion', 'able', 'ible', 'ful', 'less', 'ous', 'ive',
|
||||
'al', 'ial', 's', 'es', "'s"]
|
||||
|
||||
suffix_derivable = set()
|
||||
for k, v in multi_stroke.items():
|
||||
v_clean = v.strip().lower()
|
||||
for suffix in common_suffixes:
|
||||
if v_clean.endswith(suffix):
|
||||
base = v_clean[:-len(suffix)]
|
||||
if base in single_translations or (base + 'e') in single_translations:
|
||||
suffix_derivable.add(k)
|
||||
break
|
||||
|
||||
# Remaining after suffix rules
|
||||
remaining_dict = {k: v for k, v in raw_dict.items() if k not in suffix_derivable}
|
||||
remaining_entries = []
|
||||
for stroke_str in sorted(remaining_dict.keys()):
|
||||
strokes = tuple(parse_stroke(s) for s in stroke_str.split('/'))
|
||||
remaining_entries.append((strokes, remaining_dict[stroke_str]))
|
||||
remaining_entries.sort(key=lambda x: x[0])
|
||||
|
||||
n_remaining = len(remaining_entries)
|
||||
|
||||
# Strategy 2: Compressed page table for remaining
|
||||
all_trans = sorted(set(v for _, v in remaining_entries))
|
||||
trans_to_id = {t: i for i, t in enumerate(all_trans)}
|
||||
raw_strings = b'\x00'.join(t.encode('utf-8') for t in all_trans)
|
||||
|
||||
# String table (block-compressed)
|
||||
str_blocks = []
|
||||
for i in range(0, len(raw_strings), 4096):
|
||||
str_blocks.append(zlib.compress(raw_strings[i:i+4096], 9))
|
||||
str_total = sum(len(b) for b in str_blocks) + len(str_blocks) * 4
|
||||
|
||||
# Entries as compressed pages
|
||||
page_size = 128
|
||||
n_pages = math.ceil(n_remaining / page_size)
|
||||
total_pages_compressed = 0
|
||||
|
||||
for p in range(n_pages):
|
||||
start = p * page_size
|
||||
end = min(start + page_size, n_remaining)
|
||||
page = remaining_entries[start:end]
|
||||
|
||||
buf = bytearray()
|
||||
prev_strokes = ()
|
||||
for strokes, translation in page:
|
||||
tid = trans_to_id[translation]
|
||||
shared = 0
|
||||
for i in range(min(len(strokes), len(prev_strokes))):
|
||||
if strokes[i] == prev_strokes[i]:
|
||||
shared += 1
|
||||
else:
|
||||
break
|
||||
n_new = len(strokes) - shared
|
||||
buf.append(shared)
|
||||
buf.append(n_new)
|
||||
for s in strokes[shared:]:
|
||||
buf.extend(stroke_to_bytes(s))
|
||||
buf.extend(struct.pack('<H', tid & 0xFFFF))
|
||||
prev_strokes = strokes
|
||||
|
||||
total_pages_compressed += len(zlib.compress(bytes(buf), 9))
|
||||
|
||||
page_index = n_pages * 9
|
||||
|
||||
# Suffix rule table: store which suffix strokes map to which suffixes
|
||||
# ~20 rules × ~8 bytes = ~160 bytes
|
||||
suffix_rules_size = len(common_suffixes) * 8
|
||||
|
||||
# Base word lookup: need to know single-stroke → translation mapping
|
||||
# This is a subset of the full dict, already included in remaining_entries
|
||||
# (single-stroke entries are NOT removed by suffix rules)
|
||||
|
||||
total_hybrid = total_pages_compressed + page_index + str_total + suffix_rules_size
|
||||
|
||||
print(f" Suffix rules remove: {len(suffix_derivable)} entries")
|
||||
print(f" Remaining entries: {n_remaining}")
|
||||
print(f" Components:")
|
||||
print(f" Suffix rules: {suffix_rules_size/1024:.2f} KB")
|
||||
print(f" Entry pages: {total_pages_compressed/1024:.1f} KB ({n_pages} pages)")
|
||||
print(f" Page index: {page_index/1024:.1f} KB")
|
||||
print(f" String table: {str_total/1024:.1f} KB")
|
||||
print(f" TOTAL: {total_hybrid/1024:.1f} KB")
|
||||
print()
|
||||
|
||||
# Full zlib strings variant
|
||||
str_zlib = len(zlib.compress(raw_strings, 9))
|
||||
total_hybrid_zlib = total_pages_compressed + page_index + str_zlib + suffix_rules_size
|
||||
print(f" TOTAL (full zlib str): {total_hybrid_zlib/1024:.1f} KB")
|
||||
print(f" (needs ~4KB RAM for decompressing string blocks)")
|
||||
print()
|
||||
|
||||
return total_hybrid, total_hybrid_zlib
|
||||
|
||||
|
||||
def main():
|
||||
dict_path = '/tmp/plover-main.json'
|
||||
with open(dict_path) as f:
|
||||
raw_dict = json.load(f)
|
||||
|
||||
parsed = []
|
||||
for stroke_str in sorted(raw_dict.keys()):
|
||||
strokes = tuple(parse_stroke(s) for s in stroke_str.split('/'))
|
||||
parsed.append((strokes, raw_dict[stroke_str]))
|
||||
parsed.sort(key=lambda x: x[0])
|
||||
|
||||
print(f"Plover: {len(raw_dict)} entries")
|
||||
print()
|
||||
|
||||
# Run all approaches
|
||||
test_compressed_pages(parsed)
|
||||
test_two_level_hash(parsed)
|
||||
full_zlib, sectored = test_full_compressed(parsed)
|
||||
remaining = test_rule_reduction(raw_dict)
|
||||
hybrid, hybrid_zlib = test_hybrid(parsed, raw_dict)
|
||||
|
||||
# Summary
|
||||
print("=" * 60)
|
||||
print("FINAL SUMMARY — ALL APPROACHES")
|
||||
print("=" * 60)
|
||||
print(f" Target: 300 KB")
|
||||
print()
|
||||
|
||||
results = [
|
||||
("Compressed pages (128/pg)", None), # printed inline
|
||||
("Two-level hash", None),
|
||||
("Full zlib blob", full_zlib),
|
||||
("Sectored (4KB)", sectored),
|
||||
("Hybrid (suffix rules + pages)", hybrid),
|
||||
("Hybrid (suffix + full zlib)", hybrid_zlib),
|
||||
]
|
||||
|
||||
for name, size in results:
|
||||
if size:
|
||||
kb = size / 1024
|
||||
marker = " ✓ FITS!" if kb <= 300 else f" ({kb-300:+.0f} KB over)"
|
||||
print(f" {name:40s}: {kb:8.1f} KB{marker}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
687
tools/compression_benchmark.py
Normal file
687
tools/compression_benchmark.py
Normal file
|
|
@ -0,0 +1,687 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Benchmark compression approaches for steno dictionaries.
|
||||
|
||||
Downloads Plover main.json and measures actual output sizes for:
|
||||
1. Bit-packed DAWG (smhanov style)
|
||||
2. MPHF + block-compressed values
|
||||
3. LOUDS succinct trie
|
||||
4. Computed entries (rules + exceptions)
|
||||
5. Hybrid approaches
|
||||
"""
|
||||
|
||||
import json
|
||||
import struct
|
||||
import sys
|
||||
import zlib
|
||||
import math
|
||||
import os
|
||||
import urllib.request
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
# ─── Steno stroke parsing ───
|
||||
|
||||
STENO_KEYS = {
|
||||
'#': 0x00400000,
|
||||
'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004,
|
||||
'P-': 0x00000008, 'W-': 0x00000010, 'H-': 0x00000020,
|
||||
'R-': 0x00000040, 'A-': 0x00000080, 'O-': 0x00000100,
|
||||
'*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800,
|
||||
'-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000,
|
||||
'-B': 0x00008000, '-L': 0x00010000, '-G': 0x00020000,
|
||||
'-T': 0x00040000, '-S': 0x00080000, '-D': 0x00100000,
|
||||
'-Z': 0x00200000,
|
||||
}
|
||||
|
||||
IMPLICIT_HYPHEN = set('AOEU*')
|
||||
|
||||
def parse_stroke(s):
|
||||
result = 0
|
||||
if '#' in s:
|
||||
result |= STENO_KEYS['#']
|
||||
s = s.replace('#', '')
|
||||
has_hyphen = '-' in s
|
||||
s_clean = s.replace('-', '')
|
||||
if not has_hyphen and not any(c in IMPLICIT_HYPHEN for c in s_clean):
|
||||
for c in s_clean:
|
||||
key = c + '-'
|
||||
if key in STENO_KEYS:
|
||||
result |= STENO_KEYS[key]
|
||||
return result
|
||||
past_vowels = False
|
||||
for c in s_clean:
|
||||
if c in 'AO':
|
||||
result |= STENO_KEYS[c + '-']
|
||||
past_vowels = True
|
||||
elif c in 'EU':
|
||||
result |= STENO_KEYS['-' + c]
|
||||
past_vowels = True
|
||||
elif c == '*':
|
||||
result |= STENO_KEYS['*']
|
||||
past_vowels = True
|
||||
elif not past_vowels and (c + '-') in STENO_KEYS:
|
||||
result |= STENO_KEYS[c + '-']
|
||||
elif past_vowels and ('-' + c) in STENO_KEYS:
|
||||
result |= STENO_KEYS['-' + c]
|
||||
elif has_hyphen:
|
||||
if s.index(c) < s.index('-'):
|
||||
result |= STENO_KEYS.get(c + '-', 0)
|
||||
else:
|
||||
result |= STENO_KEYS.get('-' + c, 0)
|
||||
else:
|
||||
if (c + '-') in STENO_KEYS:
|
||||
result |= STENO_KEYS[c + '-']
|
||||
return result
|
||||
|
||||
|
||||
# ─── String table compression ───
|
||||
|
||||
def build_string_table_raw(translations):
|
||||
"""Deduplicated null-terminated strings."""
|
||||
unique = sorted(set(translations))
|
||||
table = b'\x00'.join(v.encode('utf-8') for v in unique)
|
||||
idx_map = {}
|
||||
offset = 0
|
||||
for v in unique:
|
||||
idx_map[v] = offset
|
||||
offset += len(v.encode('utf-8')) + 1
|
||||
return table, idx_map
|
||||
|
||||
def build_string_table_block_compressed(translations, block_size=4096):
|
||||
"""Block-compressed string table with random access."""
|
||||
unique = sorted(set(translations))
|
||||
raw = b'\x00'.join(v.encode('utf-8') for v in unique)
|
||||
|
||||
blocks = []
|
||||
block_offsets = []
|
||||
strings_per_block = []
|
||||
raw_offset = 0
|
||||
|
||||
for i in range(0, len(raw), block_size):
|
||||
block = raw[i:i+block_size]
|
||||
compressed = zlib.compress(block, 9)
|
||||
block_offsets.append(len(b''.join(blocks)) if blocks else 0)
|
||||
blocks.append(compressed)
|
||||
count = block.count(b'\x00') + (1 if i == 0 else 0)
|
||||
strings_per_block.append(count)
|
||||
|
||||
total_compressed = sum(len(b) for b in blocks)
|
||||
index_size = len(blocks) * 4 # block offsets
|
||||
cumulative_counts = len(blocks) * 4 # cumulative string counts
|
||||
|
||||
return total_compressed, index_size, cumulative_counts, len(unique)
|
||||
|
||||
def build_string_table_front_coded(translations):
|
||||
"""Front-coded sorted strings."""
|
||||
unique = sorted(set(translations))
|
||||
total = 0
|
||||
prev = b''
|
||||
for v in unique:
|
||||
vb = v.encode('utf-8')
|
||||
shared = 0
|
||||
for i in range(min(len(prev), len(vb))):
|
||||
if prev[i] == vb[i]:
|
||||
shared += 1
|
||||
else:
|
||||
break
|
||||
total += 2 + len(vb) - shared # prefix_len + suffix_len + suffix
|
||||
prev = vb
|
||||
return total, len(unique)
|
||||
|
||||
|
||||
# ─── Approach 1: DAWG (Daciuk algorithm) ───
|
||||
|
||||
class DawgNode:
|
||||
next_id = 0
|
||||
def __init__(self):
|
||||
self.id = DawgNode.next_id
|
||||
DawgNode.next_id += 1
|
||||
self.edges = {} # stroke_value -> DawgNode
|
||||
self.final = False
|
||||
self.count = 0 # reachable end nodes
|
||||
|
||||
def __hash__(self):
|
||||
return hash((self.final, tuple(sorted((k, v.id) for k, v in self.edges.items()))))
|
||||
|
||||
def __eq__(self, other):
|
||||
return (self.final == other.final and
|
||||
len(self.edges) == len(other.edges) and
|
||||
all(k in other.edges and self.edges[k].id == other.edges[k].id
|
||||
for k in self.edges))
|
||||
|
||||
def build_dawg(entries):
|
||||
"""Build DAWG using Daciuk's algorithm. Entries must be sorted."""
|
||||
DawgNode.next_id = 0
|
||||
root = DawgNode()
|
||||
unchecked = [] # (parent, stroke, child)
|
||||
minimized = {}
|
||||
prev_strokes = []
|
||||
|
||||
def minimize(down_to):
|
||||
for i in range(len(unchecked) - 1, down_to - 1, -1):
|
||||
parent, stroke, child = unchecked[i]
|
||||
key = (child.final, tuple(sorted((k, v.id) for k, v in child.edges.items())))
|
||||
if key in minimized:
|
||||
parent.edges[stroke] = minimized[key]
|
||||
else:
|
||||
minimized[key] = child
|
||||
unchecked.pop()
|
||||
|
||||
for strokes, _ in entries:
|
||||
# Find common prefix
|
||||
common = 0
|
||||
for i in range(min(len(strokes), len(prev_strokes))):
|
||||
if strokes[i] != prev_strokes[i]:
|
||||
break
|
||||
common += 1
|
||||
else:
|
||||
common = min(len(strokes), len(prev_strokes))
|
||||
|
||||
minimize(common)
|
||||
|
||||
# Add suffix
|
||||
if unchecked:
|
||||
node = unchecked[-1][2]
|
||||
else:
|
||||
node = root
|
||||
|
||||
for stroke in strokes[common:]:
|
||||
new_node = DawgNode()
|
||||
node.edges[stroke] = new_node
|
||||
unchecked.append((node, stroke, new_node))
|
||||
node = new_node
|
||||
|
||||
node.final = True
|
||||
prev_strokes = strokes
|
||||
|
||||
minimize(0)
|
||||
|
||||
# Count reachable end nodes for each node
|
||||
def count_reachable(node, visited=None):
|
||||
if visited is None:
|
||||
visited = {}
|
||||
if node.id in visited:
|
||||
return visited[node.id]
|
||||
c = 1 if node.final else 0
|
||||
for child in node.edges.values():
|
||||
c += count_reachable(child, visited)
|
||||
visited[node.id] = c
|
||||
node.count = c
|
||||
return c
|
||||
|
||||
count_reachable(root)
|
||||
return root
|
||||
|
||||
def measure_dawg(root):
|
||||
"""Count nodes, edges, measure bit-packed size."""
|
||||
nodes = set()
|
||||
edges = 0
|
||||
fallthrough = 0
|
||||
child_dist = Counter()
|
||||
|
||||
def visit(node):
|
||||
nonlocal edges, fallthrough
|
||||
if node.id in nodes:
|
||||
return
|
||||
nodes.add(node.id)
|
||||
n_children = len(node.edges)
|
||||
child_dist[n_children] += 1
|
||||
edges += n_children
|
||||
if n_children == 1:
|
||||
fallthrough += 1
|
||||
for child in node.edges.values():
|
||||
visit(child)
|
||||
|
||||
visit(root)
|
||||
return len(nodes), edges, fallthrough, child_dist
|
||||
|
||||
def estimate_dawg_bitpacked(n_nodes, n_edges, n_fallthrough, unique_strokes, n_entries):
|
||||
"""Estimate bit-packed DAWG size (smhanov format)."""
|
||||
cbits = max(1, math.ceil(math.log2(max(unique_strokes, 2))))
|
||||
abits = max(1, math.ceil(math.log2(max(n_nodes, 2))))
|
||||
nskipbits = max(1, math.ceil(math.log2(max(n_entries, 2))))
|
||||
|
||||
# Fallthrough nodes: 2 + cbits bits
|
||||
ft_bits = n_fallthrough * (2 + cbits)
|
||||
# Leaf nodes (0 children): 2 bits
|
||||
n_leaf = sum(1 for _ in range(n_nodes) if True) # approximate
|
||||
# Multi-edge nodes: 2 + 1 + n_children * (cbits + nskipbits + abits)
|
||||
non_ft_edges = n_edges - n_fallthrough
|
||||
multi_bits = (n_nodes - n_fallthrough) * 3 # header per non-fallthrough
|
||||
multi_bits += non_ft_edges * (cbits + nskipbits + abits)
|
||||
|
||||
total_bits = ft_bits + multi_bits
|
||||
return total_bits // 8, cbits, abits, nskipbits
|
||||
|
||||
|
||||
# ─── Approach 2: MPHF + values ───
|
||||
|
||||
def estimate_mphf(n_entries, n_unique_strokes):
|
||||
"""Estimate MPHF-based approach size."""
|
||||
mphf_bits_per_key = 2.5 # CHD or similar
|
||||
mphf_bytes = int(n_entries * mphf_bits_per_key / 8)
|
||||
fingerprint_bytes = n_entries * 2 # 16-bit fingerprints
|
||||
return mphf_bytes, fingerprint_bytes
|
||||
|
||||
|
||||
# ─── Approach 3: LOUDS succinct trie ───
|
||||
|
||||
def build_louds_trie(entries):
|
||||
"""Build trie and encode as LOUDS."""
|
||||
# Build trie
|
||||
class TrieNode:
|
||||
__slots__ = ['children', 'is_end']
|
||||
def __init__(self):
|
||||
self.children = {}
|
||||
self.is_end = False
|
||||
|
||||
root = TrieNode()
|
||||
for strokes, _ in entries:
|
||||
node = root
|
||||
for s in strokes:
|
||||
if s not in node.children:
|
||||
node.children[s] = TrieNode()
|
||||
node = node.children[s]
|
||||
node.is_end = True
|
||||
|
||||
# BFS to build LOUDS
|
||||
from collections import deque
|
||||
queue = deque([root])
|
||||
louds_bits = [] # 1 per child, 0 as separator
|
||||
labels = []
|
||||
is_final = []
|
||||
n_nodes = 0
|
||||
|
||||
# Super root
|
||||
louds_bits.append(1) # root is child of super root
|
||||
louds_bits.append(0)
|
||||
|
||||
while queue:
|
||||
node = queue.popleft()
|
||||
n_nodes += 1
|
||||
is_final.append(node.is_end)
|
||||
for stroke in sorted(node.children.keys()):
|
||||
louds_bits.append(1)
|
||||
labels.append(stroke)
|
||||
queue.append(node.children[stroke])
|
||||
louds_bits.append(0) # separator
|
||||
|
||||
return louds_bits, labels, is_final, n_nodes
|
||||
|
||||
def measure_louds(louds_bits, labels, is_final, n_nodes, unique_strokes, n_entries):
|
||||
"""Measure LOUDS encoding size."""
|
||||
# LOUDS bitvector
|
||||
louds_bytes = (len(louds_bits) + 7) // 8
|
||||
# Rank/select auxiliary structures (~37.5% overhead for practical implementations)
|
||||
rank_select_bytes = int(louds_bytes * 0.375)
|
||||
# Labels: each label = stroke value
|
||||
cbits = max(1, math.ceil(math.log2(max(unique_strokes, 2))))
|
||||
labels_bytes = (len(labels) * cbits + 7) // 8
|
||||
# is_final bitvector
|
||||
final_bytes = (n_nodes + 7) // 8
|
||||
final_rank_bytes = int(final_bytes * 0.375)
|
||||
|
||||
return {
|
||||
'louds_bitvec': louds_bytes,
|
||||
'rank_select': rank_select_bytes,
|
||||
'labels': labels_bytes,
|
||||
'is_final': final_bytes + final_rank_bytes,
|
||||
'total': louds_bytes + rank_select_bytes + labels_bytes + final_bytes + final_rank_bytes,
|
||||
}
|
||||
|
||||
|
||||
# ─── Approach 4: Computed entries analysis ───
|
||||
|
||||
def analyze_computed_entries(entries_dict):
|
||||
"""Analyze how many entries follow computable patterns."""
|
||||
computable = 0
|
||||
rule_categories = Counter()
|
||||
|
||||
for stroke_str, translation in entries_dict.items():
|
||||
# Fingerspelling: single letter output from specific strokes
|
||||
if len(translation) == 1 and translation.isalpha():
|
||||
computable += 1
|
||||
rule_categories['fingerspelling'] += 1
|
||||
continue
|
||||
|
||||
# Number entries: output is digits
|
||||
if translation.replace(',', '').replace('.', '').replace('-', '').isdigit():
|
||||
computable += 1
|
||||
rule_categories['numbers'] += 1
|
||||
continue
|
||||
|
||||
# Simple suffix entries: {^ing}, {^ed}, {^ly}, {^er}, {^ment}, {^ness}
|
||||
if translation.startswith('{^') and translation.endswith('}'):
|
||||
suffix = translation[2:-1]
|
||||
if suffix in ('ing', 'ed', 'ly', 'er', 'est', 'ment', 'ness', 'tion',
|
||||
'sion', 'able', 'ible', 'ful', 'less', 'ous', 'ive',
|
||||
'al', 'ial', 'en', 'ize', 'ise', 'ity', 'ty',
|
||||
's', 'es', "'s", 'ry', 'ary'):
|
||||
computable += 1
|
||||
rule_categories['common_suffix'] += 1
|
||||
continue
|
||||
|
||||
# Simple prefix entries: {pre^}, {re^}, {un^}
|
||||
if translation.startswith('{') and translation.endswith('^}'):
|
||||
prefix = translation[1:-2]
|
||||
if prefix in ('re', 'un', 'pre', 'dis', 'mis', 'over', 'under',
|
||||
'out', 'sub', 'super', 'anti', 'auto', 'bi', 'co',
|
||||
'de', 'ex', 'inter', 'macro', 'micro', 'mid', 'mini',
|
||||
'mono', 'multi', 'non', 'post', 'semi', 'tri'):
|
||||
computable += 1
|
||||
rule_categories['common_prefix'] += 1
|
||||
continue
|
||||
|
||||
# Plover commands: {#...}, {PLOVER:...}, {MODE:...}
|
||||
if translation.startswith('{#') or translation.startswith('{PLOVER:') or \
|
||||
translation.startswith('{MODE:'):
|
||||
computable += 1
|
||||
rule_categories['commands'] += 1
|
||||
continue
|
||||
|
||||
# Punctuation/formatting: {.}, {,}, {?}, {!}, {^}, {-|}
|
||||
if translation in ('{.}', '{,}', '{?}', '{!}', '{^}', '{-|}', '{*-|}',
|
||||
'{*!}', '{*?}', '{<}', '{>}', '{*<}', '{*>}',
|
||||
'{^~|^}', '{~|}'):
|
||||
computable += 1
|
||||
rule_categories['formatting'] += 1
|
||||
continue
|
||||
|
||||
return computable, rule_categories
|
||||
|
||||
|
||||
# ─── Approach 5: FST-style encoding ───
|
||||
|
||||
def estimate_fst(entries, unique_strokes):
|
||||
"""
|
||||
FST shares both prefixes AND suffixes on OUTPUT side too.
|
||||
Output = sequence of output tokens along edges.
|
||||
"""
|
||||
# In an FST, each edge carries an output fragment
|
||||
# Common output prefixes/suffixes are shared
|
||||
# For steno: input = stroke sequence, output = translation
|
||||
|
||||
# Build input trie first (same as DAWG input)
|
||||
# Then attach output weights to edges
|
||||
# FST minimization merges states with identical futures (like DAWG)
|
||||
# PLUS merges output-compatible states
|
||||
|
||||
# Estimate: FST typically achieves 2-4 bytes per entry for English word lists
|
||||
# For steno with longer outputs, maybe 4-8 bytes per entry
|
||||
|
||||
# Use BurntSushi/fst benchmarks as reference:
|
||||
# 235K English words → ~750KB FST
|
||||
# That's ~3.2 bytes per entry
|
||||
|
||||
# For steno: 147K entries, but outputs are longer (avg 8.6 chars vs 7 for English)
|
||||
# Rough: 4-6 bytes per entry
|
||||
low = len(entries) * 4
|
||||
high = len(entries) * 6
|
||||
return low, high
|
||||
|
||||
|
||||
# ─── Main benchmark ───
|
||||
|
||||
def main():
|
||||
# Download Plover dict
|
||||
dict_path = '/tmp/plover-main.json'
|
||||
if not os.path.exists(dict_path):
|
||||
print("Downloading Plover main.json...")
|
||||
urllib.request.urlretrieve(
|
||||
"https://raw.githubusercontent.com/openstenoproject/plover/main/plover/assets/main.json",
|
||||
dict_path)
|
||||
|
||||
with open(dict_path) as f:
|
||||
raw_dict = json.load(f)
|
||||
|
||||
print(f"Plover main.json: {len(raw_dict)} entries")
|
||||
print()
|
||||
|
||||
# Parse all strokes
|
||||
parsed = []
|
||||
unique_stroke_vals = set()
|
||||
for stroke_str in sorted(raw_dict.keys()):
|
||||
strokes = tuple(parse_stroke(s) for s in stroke_str.split('/'))
|
||||
for s in strokes:
|
||||
unique_stroke_vals.add(s)
|
||||
parsed.append((strokes, raw_dict[stroke_str]))
|
||||
|
||||
# Sort by stroke tuple for DAWG construction
|
||||
parsed.sort(key=lambda x: x[0])
|
||||
|
||||
n_entries = len(parsed)
|
||||
translations = [v for _, v in parsed]
|
||||
unique_translations = set(translations)
|
||||
n_unique_strokes = len(unique_stroke_vals)
|
||||
|
||||
print(f"Unique stroke values: {n_unique_strokes}")
|
||||
print(f"Unique translations: {len(unique_translations)}")
|
||||
print()
|
||||
|
||||
# ─── String table measurements ───
|
||||
print("=" * 60)
|
||||
print("STRING TABLE OPTIONS")
|
||||
print("=" * 60)
|
||||
|
||||
raw_table, _ = build_string_table_raw(translations)
|
||||
print(f" Raw deduplicated: {len(raw_table)/1024:8.1f} KB")
|
||||
|
||||
fc_size, fc_count = build_string_table_front_coded(translations)
|
||||
print(f" Front-coded: {fc_size/1024:8.1f} KB")
|
||||
|
||||
bc_data, bc_idx, bc_cum, bc_unique = build_string_table_block_compressed(
|
||||
translations, block_size=4096)
|
||||
bc_total = bc_data + bc_idx + bc_cum
|
||||
print(f" Block-compressed 4KB: {bc_total/1024:8.1f} KB (data={bc_data/1024:.1f} idx={bc_idx/1024:.1f})")
|
||||
|
||||
bc_data2, bc_idx2, bc_cum2, _ = build_string_table_block_compressed(
|
||||
translations, block_size=2048)
|
||||
bc_total2 = bc_data2 + bc_idx2 + bc_cum2
|
||||
print(f" Block-compressed 2KB: {bc_total2/1024:8.1f} KB")
|
||||
|
||||
bc_data3, bc_idx3, bc_cum3, _ = build_string_table_block_compressed(
|
||||
translations, block_size=8192)
|
||||
bc_total3 = bc_data3 + bc_idx3 + bc_cum3
|
||||
print(f" Block-compressed 8KB: {bc_total3/1024:8.1f} KB")
|
||||
|
||||
# Full zlib (no random access)
|
||||
full_zlib = len(zlib.compress(raw_table, 9))
|
||||
print(f" Full zlib (no RA): {full_zlib/1024:8.1f} KB")
|
||||
|
||||
# Value index: maps entry index → string table position
|
||||
val_idx_2b = n_entries * 2
|
||||
val_idx_3b = n_entries * 3
|
||||
# With dedup: entry → unique_string_id (17 bits for 70K)
|
||||
dedup_idx_bits = n_entries * math.ceil(math.log2(len(unique_translations)))
|
||||
dedup_idx_bytes = (dedup_idx_bits + 7) // 8
|
||||
print(f" Value index (2B/ent): {val_idx_2b/1024:8.1f} KB")
|
||||
print(f" Value index (dedup): {dedup_idx_bytes/1024:8.1f} KB ({math.ceil(math.log2(len(unique_translations)))} bits/ent)")
|
||||
print()
|
||||
|
||||
# ─── Approach 1: DAWG ───
|
||||
print("=" * 60)
|
||||
print("APPROACH 1: BIT-PACKED DAWG")
|
||||
print("=" * 60)
|
||||
|
||||
print(" Building DAWG (may take ~30s)...")
|
||||
root = build_dawg(parsed)
|
||||
n_nodes, n_edges, n_fallthrough, child_dist = measure_dawg(root)
|
||||
print(f" Nodes: {n_nodes}")
|
||||
print(f" Edges: {n_edges}")
|
||||
print(f" Fallthrough (1-child): {n_fallthrough}")
|
||||
print(f" Suffix dedup: {(1 - n_nodes/184582)*100:.1f}% reduction from trie")
|
||||
|
||||
dawg_bytes, cbits, abits, nskipbits = estimate_dawg_bitpacked(
|
||||
n_nodes, n_edges, n_fallthrough, n_unique_strokes, n_entries)
|
||||
print(f" cbits={cbits} abits={abits} nskipbits={nskipbits}")
|
||||
print(f" DAWG structure: {dawg_bytes/1024:8.1f} KB")
|
||||
|
||||
# DAWG gives implicit index via skip counts → no separate value index needed
|
||||
# Total = DAWG + string table
|
||||
dawg_total = dawg_bytes + bc_total
|
||||
print(f" + block-compressed strings: {bc_total/1024:.1f} KB")
|
||||
print(f" TOTAL (DAWG): {dawg_total/1024:8.1f} KB")
|
||||
print()
|
||||
|
||||
# ─── Approach 2: MPHF ───
|
||||
print("=" * 60)
|
||||
print("APPROACH 2: MPHF + BLOCK-COMPRESSED VALUES")
|
||||
print("=" * 60)
|
||||
|
||||
mphf_bytes, fp_bytes = estimate_mphf(n_entries, n_unique_strokes)
|
||||
print(f" MPHF (~2.5 bits/key): {mphf_bytes/1024:8.1f} KB")
|
||||
print(f" Fingerprints (16-bit): {fp_bytes/1024:8.1f} KB")
|
||||
print(f" Fingerprints (8-bit): {fp_bytes/2/1024:8.1f} KB")
|
||||
# Need to store stroke sequences for fingerprint verification
|
||||
# Average stroke seq: 2.3 strokes × 3 bytes = 6.9 bytes per key
|
||||
stroke_storage = int(n_entries * 2.3 * 3)
|
||||
print(f" Stroke key storage: {stroke_storage/1024:8.1f} KB (for verification)")
|
||||
|
||||
mphf_total_16 = mphf_bytes + fp_bytes + bc_total + dedup_idx_bytes
|
||||
mphf_total_8 = mphf_bytes + fp_bytes // 2 + bc_total + dedup_idx_bytes
|
||||
mphf_total_nofp = mphf_bytes + bc_total + dedup_idx_bytes # no fingerprint, accept false positives
|
||||
print(f" TOTAL (16-bit fp): {mphf_total_16/1024:8.1f} KB")
|
||||
print(f" TOTAL (8-bit fp): {mphf_total_8/1024:8.1f} KB")
|
||||
print(f" TOTAL (no fp): {mphf_total_nofp/1024:8.1f} KB (0.4% false positive)")
|
||||
print()
|
||||
|
||||
# ─── Approach 3: LOUDS ───
|
||||
print("=" * 60)
|
||||
print("APPROACH 3: LOUDS SUCCINCT TRIE")
|
||||
print("=" * 60)
|
||||
|
||||
print(" Building LOUDS trie...")
|
||||
louds_bits, labels, is_final, louds_n_nodes = build_louds_trie(parsed)
|
||||
louds_sizes = measure_louds(louds_bits, labels, is_final, louds_n_nodes,
|
||||
n_unique_strokes, n_entries)
|
||||
for k, v in louds_sizes.items():
|
||||
if k != 'total':
|
||||
print(f" {k:20s}: {v/1024:8.1f} KB")
|
||||
louds_total = louds_sizes['total'] + bc_total + dedup_idx_bytes
|
||||
print(f" + strings + val index: {(bc_total + dedup_idx_bytes)/1024:.1f} KB")
|
||||
print(f" TOTAL (LOUDS): {louds_total/1024:8.1f} KB")
|
||||
print()
|
||||
|
||||
# ─── Approach 4: Computed entries ───
|
||||
print("=" * 60)
|
||||
print("APPROACH 4: COMPUTED ENTRIES ANALYSIS")
|
||||
print("=" * 60)
|
||||
|
||||
computable, categories = analyze_computed_entries(raw_dict)
|
||||
remaining = n_entries - computable
|
||||
print(f" Computable entries: {computable} ({computable/n_entries*100:.1f}%)")
|
||||
for cat, count in categories.most_common():
|
||||
print(f" {cat:20s}: {count}")
|
||||
print(f" Remaining (stored): {remaining}")
|
||||
print(f" If remaining used DAWG approach:")
|
||||
reduction = remaining / n_entries
|
||||
computed_dawg_est = dawg_total * reduction
|
||||
print(f" Estimated: {computed_dawg_est/1024:8.1f} KB")
|
||||
print()
|
||||
|
||||
# ─── Approach 5: FST estimate ───
|
||||
print("=" * 60)
|
||||
print("APPROACH 5: FST (FINITE STATE TRANSDUCER) ESTIMATE")
|
||||
print("=" * 60)
|
||||
|
||||
fst_low, fst_high = estimate_fst(parsed, n_unique_strokes)
|
||||
print(f" FST (4 bytes/entry): {fst_low/1024:8.1f} KB")
|
||||
print(f" FST (6 bytes/entry): {fst_high/1024:8.1f} KB")
|
||||
print(f" Note: FST stores keys + values together, no separate string table")
|
||||
print()
|
||||
|
||||
# ─── Hybrid approaches ───
|
||||
print("=" * 60)
|
||||
print("HYBRID APPROACHES")
|
||||
print("=" * 60)
|
||||
|
||||
# Hybrid 1: Computed entries + DAWG for rest
|
||||
print(f"\n HYBRID 1: Computed rules + DAWG for remaining {remaining} entries")
|
||||
h1_rules = 5 # KB for rule engine code
|
||||
h1_dawg = dawg_total * reduction
|
||||
h1_total = h1_rules * 1024 + h1_dawg
|
||||
print(f" Rules engine: {h1_rules:8.1f} KB")
|
||||
print(f" DAWG (remaining): {h1_dawg/1024:8.1f} KB")
|
||||
print(f" TOTAL: {h1_total/1024:8.1f} KB")
|
||||
|
||||
# Hybrid 2: MPHF (no fingerprint) + full zlib strings + dedup index
|
||||
print(f"\n HYBRID 2: MPHF + full zlib (decompress to RAM per-block)")
|
||||
h2_total = mphf_bytes + bc_total + dedup_idx_bytes
|
||||
print(f" MPHF: {mphf_bytes/1024:8.1f} KB")
|
||||
print(f" Strings (block): {bc_total/1024:8.1f} KB")
|
||||
print(f" Value index (dedup): {dedup_idx_bytes/1024:8.1f} KB")
|
||||
print(f" TOTAL: {h2_total/1024:8.1f} KB")
|
||||
|
||||
# Hybrid 3: LOUDS trie (no value index needed - use rank on is_final)
|
||||
# The rank of the final-bit gives the entry index
|
||||
print(f"\n HYBRID 3: LOUDS + rank-based indexing (no value index array)")
|
||||
h3_total = louds_sizes['total'] + bc_total
|
||||
print(f" LOUDS structure: {louds_sizes['total']/1024:8.1f} KB")
|
||||
print(f" Strings (block): {bc_total/1024:8.1f} KB")
|
||||
print(f" TOTAL: {h3_total/1024:8.1f} KB")
|
||||
|
||||
# Hybrid 4: Computed + LOUDS for remaining
|
||||
print(f"\n HYBRID 4: Computed + LOUDS for remaining {remaining}")
|
||||
h4_louds_est = louds_sizes['total'] * reduction
|
||||
h4_strings_est = bc_total * reduction
|
||||
h4_total = h1_rules * 1024 + h4_louds_est + h4_strings_est
|
||||
print(f" Rules engine: {h1_rules:8.1f} KB")
|
||||
print(f" LOUDS (remaining): {h4_louds_est/1024:8.1f} KB")
|
||||
print(f" Strings (remaining): {h4_strings_est/1024:8.1f} KB")
|
||||
print(f" TOTAL: {h4_total/1024:8.1f} KB")
|
||||
|
||||
# Hybrid 5: Computed + MPHF for remaining (no fingerprint)
|
||||
print(f"\n HYBRID 5: Computed + MPHF for remaining {remaining}")
|
||||
h5_mphf = int(remaining * 2.5 / 8)
|
||||
h5_strings = int(bc_total * reduction)
|
||||
h5_dedup = int(dedup_idx_bytes * reduction)
|
||||
h5_total = h1_rules * 1024 + h5_mphf + h5_strings + h5_dedup
|
||||
print(f" Rules engine: {h1_rules:8.1f} KB")
|
||||
print(f" MPHF (remaining): {h5_mphf/1024:8.1f} KB")
|
||||
print(f" Strings (remaining): {h5_strings/1024:8.1f} KB")
|
||||
print(f" Value index: {h5_dedup/1024:8.1f} KB")
|
||||
print(f" TOTAL: {h5_total/1024:8.1f} KB")
|
||||
|
||||
# Hybrid 6: DAWG keys (implicit indexing) + zlib strings with smaller blocks
|
||||
print(f"\n HYBRID 6: DAWG (implicit index) + aggressive string compression")
|
||||
# Use DAWG skip-count for indexing (no value array)
|
||||
# Try smaller zlib blocks for better compression at cost of more overhead
|
||||
bc_data_1k, bc_idx_1k, bc_cum_1k, _ = build_string_table_block_compressed(
|
||||
translations, block_size=1024)
|
||||
bc_total_1k = bc_data_1k + bc_idx_1k + bc_cum_1k
|
||||
print(f" DAWG structure: {dawg_bytes/1024:8.1f} KB")
|
||||
print(f" Strings (1KB block): {bc_total_1k/1024:8.1f} KB")
|
||||
print(f" TOTAL: {(dawg_bytes + bc_total_1k)/1024:8.1f} KB")
|
||||
|
||||
print()
|
||||
print("=" * 60)
|
||||
print("SUMMARY — ALL APPROACHES RANKED BY SIZE")
|
||||
print("=" * 60)
|
||||
print(f" Target: 300 KB")
|
||||
print()
|
||||
|
||||
approaches = [
|
||||
("DAWG + block strings", dawg_total),
|
||||
("MPHF no-fp + strings", mphf_total_nofp),
|
||||
("MPHF 8-bit fp", mphf_total_8),
|
||||
("LOUDS + strings", louds_total),
|
||||
("LOUDS rank-index", h3_total),
|
||||
("FST (optimistic)", fst_low),
|
||||
("FST (conservative)", fst_high),
|
||||
("Hybrid 1: Compute+DAWG", h1_total),
|
||||
("Hybrid 2: MPHF+block", h2_total),
|
||||
("Hybrid 3: LOUDS+rank", h3_total),
|
||||
("Hybrid 4: Compute+LOUDS", h4_total),
|
||||
("Hybrid 5: Compute+MPHF", h5_total),
|
||||
("Hybrid 6: DAWG+aggr.str", dawg_bytes + bc_total_1k),
|
||||
]
|
||||
|
||||
approaches.sort(key=lambda x: x[1])
|
||||
|
||||
for name, size in approaches:
|
||||
kb = size / 1024
|
||||
marker = " ✓" if kb <= 300 else f" ({kb-300:+.0f} KB over)"
|
||||
print(f" {name:28s}: {kb:8.1f} KB{marker}")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
722
tools/dawg_fst_prototype.py
Normal file
722
tools/dawg_fst_prototype.py
Normal file
|
|
@ -0,0 +1,722 @@
|
|||
#!/usr/bin/env python3
|
||||
"""Prototype: DAWG with FST-style edge outputs.
|
||||
|
||||
DAWG handles key structure (22K nodes, 87.8% suffix sharing).
|
||||
Outputs encoded directly on edges — no separate value index.
|
||||
|
||||
When traversing stroke sequence, accumulate output fragments from edges.
|
||||
Final node's output = concatenation of all edge outputs along path.
|
||||
|
||||
String dedup: outputs reference into block-compressed string table.
|
||||
"""
|
||||
|
||||
import json
|
||||
import struct
|
||||
import sys
|
||||
import zlib
|
||||
import math
|
||||
import os
|
||||
from collections import Counter, defaultdict
|
||||
|
||||
# ─── Steno stroke parsing ───
|
||||
|
||||
STENO_KEYS = {
|
||||
'#': 0x00400000,
|
||||
'S-': 0x00000001, 'T-': 0x00000002, 'K-': 0x00000004,
|
||||
'P-': 0x00000008, 'W-': 0x00000010, 'H-': 0x00000020,
|
||||
'R-': 0x00000040, 'A-': 0x00000080, 'O-': 0x00000100,
|
||||
'*': 0x00000200, '-E': 0x00000400, '-U': 0x00000800,
|
||||
'-F': 0x00001000, '-R': 0x00002000, '-P': 0x00004000,
|
||||
'-B': 0x00008000, '-L': 0x00010000, '-G': 0x00020000,
|
||||
'-T': 0x00040000, '-S': 0x00080000, '-D': 0x00100000,
|
||||
'-Z': 0x00200000,
|
||||
}
|
||||
|
||||
IMPLICIT_HYPHEN = set('AOEU*')
|
||||
|
||||
def parse_stroke(s):
|
||||
result = 0
|
||||
if '#' in s:
|
||||
result |= STENO_KEYS['#']
|
||||
s = s.replace('#', '')
|
||||
has_hyphen = '-' in s
|
||||
s_clean = s.replace('-', '')
|
||||
if not has_hyphen and not any(c in IMPLICIT_HYPHEN for c in s_clean):
|
||||
for c in s_clean:
|
||||
key = c + '-'
|
||||
if key in STENO_KEYS:
|
||||
result |= STENO_KEYS[key]
|
||||
return result
|
||||
past_vowels = False
|
||||
for c in s_clean:
|
||||
if c in 'AO':
|
||||
result |= STENO_KEYS[c + '-']
|
||||
past_vowels = True
|
||||
elif c in 'EU':
|
||||
result |= STENO_KEYS['-' + c]
|
||||
past_vowels = True
|
||||
elif c == '*':
|
||||
result |= STENO_KEYS['*']
|
||||
past_vowels = True
|
||||
elif not past_vowels and (c + '-') in STENO_KEYS:
|
||||
result |= STENO_KEYS[c + '-']
|
||||
elif past_vowels and ('-' + c) in STENO_KEYS:
|
||||
result |= STENO_KEYS['-' + c]
|
||||
elif has_hyphen:
|
||||
if s.index(c) < s.index('-'):
|
||||
result |= STENO_KEYS.get(c + '-', 0)
|
||||
else:
|
||||
result |= STENO_KEYS.get('-' + c, 0)
|
||||
else:
|
||||
if (c + '-') in STENO_KEYS:
|
||||
result |= STENO_KEYS[c + '-']
|
||||
return result
|
||||
|
||||
|
||||
# ─── DAWG with outputs ───
|
||||
|
||||
class DawgNode:
|
||||
next_id = 0
|
||||
def __init__(self):
|
||||
self.id = DawgNode.next_id
|
||||
DawgNode.next_id += 1
|
||||
self.edges = {} # stroke_val -> (DawgNode, output_str)
|
||||
self.final = False
|
||||
self.final_output = "" # remaining output at final state
|
||||
|
||||
def signature(self):
|
||||
"""Signature for minimization — includes outputs."""
|
||||
edge_sig = tuple(sorted(
|
||||
(k, child.id, out) for k, (child, out) in self.edges.items()
|
||||
))
|
||||
return (self.final, self.final_output if self.final else "", edge_sig)
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.signature())
|
||||
|
||||
def __eq__(self, other):
|
||||
return self.signature() == other.signature()
|
||||
|
||||
|
||||
def build_dawg_fst(entries):
|
||||
"""
|
||||
Build DAWG with FST-style outputs.
|
||||
|
||||
For each entry (strokes, translation):
|
||||
- Walk trie path for strokes
|
||||
- Attach output to FIRST edge (FST convention: push output left)
|
||||
- At final node, store remaining output
|
||||
|
||||
Then minimize: merge nodes with identical futures (including outputs).
|
||||
|
||||
This is a simplified FST construction — not fully optimal but
|
||||
captures most of the savings.
|
||||
"""
|
||||
DawgNode.next_id = 0
|
||||
|
||||
# Phase 1: Build trie with outputs
|
||||
root = DawgNode()
|
||||
|
||||
for strokes, translation in entries:
|
||||
node = root
|
||||
for i, stroke in enumerate(strokes):
|
||||
if stroke not in node.edges:
|
||||
new_node = DawgNode()
|
||||
node.edges[stroke] = (new_node, "")
|
||||
node = new_node
|
||||
else:
|
||||
node = node.edges[stroke][0]
|
||||
node.final = True
|
||||
node.final_output = translation
|
||||
|
||||
# Phase 2: Push outputs to edges (left-push)
|
||||
# For each node, find common prefix of all outputs reachable,
|
||||
# push that prefix to the incoming edge, strip from descendants.
|
||||
# This enables more suffix sharing.
|
||||
|
||||
def push_outputs(node, depth=0):
|
||||
"""Push common output prefixes toward the root."""
|
||||
if not node.edges:
|
||||
return
|
||||
|
||||
# First recurse into children
|
||||
for stroke, (child, out) in list(node.edges.items()):
|
||||
push_outputs(child, depth + 1)
|
||||
|
||||
# For each child, collect all outputs reachable from it
|
||||
for stroke, (child, edge_out) in list(node.edges.items()):
|
||||
if child.final and not child.edges:
|
||||
# Leaf: output = edge_out + final_output
|
||||
full_out = edge_out + child.final_output
|
||||
node.edges[stroke] = (child, full_out)
|
||||
child.final_output = ""
|
||||
|
||||
push_outputs(root)
|
||||
|
||||
# Phase 3: Minimize (merge identical subtrees including outputs)
|
||||
minimized = {}
|
||||
|
||||
def minimize_node(node, visited=None):
|
||||
if visited is None:
|
||||
visited = set()
|
||||
if node.id in visited:
|
||||
return node
|
||||
visited.add(node.id)
|
||||
|
||||
# First minimize children
|
||||
for stroke, (child, out) in list(node.edges.items()):
|
||||
minimized_child = minimize_node(child, visited)
|
||||
node.edges[stroke] = (minimized_child, out)
|
||||
|
||||
# Check if we've seen an equivalent node
|
||||
sig = node.signature()
|
||||
if sig in minimized:
|
||||
return minimized[sig]
|
||||
minimized[sig] = node
|
||||
return node
|
||||
|
||||
root = minimize_node(root)
|
||||
|
||||
return root
|
||||
|
||||
|
||||
def measure_dawg_fst(root):
|
||||
"""Measure the DAWG-FST structure."""
|
||||
nodes = set()
|
||||
edges = 0
|
||||
total_output_bytes = 0
|
||||
output_lengths = []
|
||||
unique_outputs = set()
|
||||
|
||||
def visit(node):
|
||||
nonlocal edges, total_output_bytes
|
||||
if node.id in nodes:
|
||||
return
|
||||
nodes.add(node.id)
|
||||
for stroke, (child, output) in node.edges.items():
|
||||
edges += 1
|
||||
out_bytes = output.encode('utf-8')
|
||||
total_output_bytes += len(out_bytes)
|
||||
output_lengths.append(len(out_bytes))
|
||||
if output:
|
||||
unique_outputs.add(output)
|
||||
visit(child)
|
||||
if node.final and node.final_output:
|
||||
total_output_bytes += len(node.final_output.encode('utf-8'))
|
||||
output_lengths.append(len(node.final_output.encode('utf-8')))
|
||||
unique_outputs.add(node.final_output)
|
||||
|
||||
visit(root)
|
||||
return {
|
||||
'nodes': len(nodes),
|
||||
'edges': edges,
|
||||
'total_output_bytes': total_output_bytes,
|
||||
'unique_outputs': len(unique_outputs),
|
||||
'avg_output_len': sum(output_lengths) / max(len(output_lengths), 1),
|
||||
'output_lengths': output_lengths,
|
||||
}
|
||||
|
||||
|
||||
def verify_dawg_fst(root, entries):
|
||||
"""Verify lookups return correct translations."""
|
||||
correct = 0
|
||||
wrong = 0
|
||||
missing = 0
|
||||
|
||||
for strokes, expected in entries:
|
||||
node = root
|
||||
output = ""
|
||||
found = True
|
||||
for stroke in strokes:
|
||||
if stroke in node.edges:
|
||||
child, edge_out = node.edges[stroke]
|
||||
output += edge_out
|
||||
node = child
|
||||
else:
|
||||
found = False
|
||||
break
|
||||
|
||||
if found and node.final:
|
||||
output += node.final_output
|
||||
if output == expected:
|
||||
correct += 1
|
||||
else:
|
||||
wrong += 1
|
||||
if wrong <= 5:
|
||||
print(f" WRONG: expected '{expected}', got '{output}'")
|
||||
else:
|
||||
missing += 1
|
||||
if missing <= 5:
|
||||
print(f" MISSING: {strokes} -> '{expected}'")
|
||||
|
||||
return correct, wrong, missing
|
||||
|
||||
|
||||
def estimate_binary_size(stats, n_unique_strokes):
|
||||
"""Estimate binary encoding size."""
|
||||
n_nodes = stats['nodes']
|
||||
n_edges = stats['edges']
|
||||
|
||||
cbits = max(1, math.ceil(math.log2(max(n_unique_strokes, 2))))
|
||||
abits = max(1, math.ceil(math.log2(max(n_nodes, 2))))
|
||||
|
||||
# Edge encoding: stroke_key + target_node + output_ref
|
||||
# output_ref: index into output string table
|
||||
unique_outputs = stats['unique_outputs']
|
||||
obits = max(1, math.ceil(math.log2(max(unique_outputs + 1, 2)))) # +1 for "no output"
|
||||
|
||||
bits_per_edge = cbits + abits + obits + 1 # +1 for last-edge flag
|
||||
total_edge_bits = n_edges * bits_per_edge
|
||||
|
||||
# Node overhead: 1 bit for is_final, 1 bit for has_final_output
|
||||
node_bits = n_nodes * 2
|
||||
|
||||
# Final outputs: nodes with final_output need an output reference
|
||||
# Approximate: ~50% of final nodes have output
|
||||
final_output_bits = n_nodes * obits // 4 # rough
|
||||
|
||||
structure_bits = total_edge_bits + node_bits + final_output_bits
|
||||
structure_bytes = (structure_bits + 7) // 8
|
||||
|
||||
return {
|
||||
'cbits': cbits,
|
||||
'abits': abits,
|
||||
'obits': obits,
|
||||
'bits_per_edge': bits_per_edge,
|
||||
'structure_bytes': structure_bytes,
|
||||
}
|
||||
|
||||
|
||||
def build_output_string_table(root):
|
||||
"""Collect all unique output strings and build compressed table."""
|
||||
outputs = set()
|
||||
|
||||
def visit(node, visited=None):
|
||||
if visited is None:
|
||||
visited = set()
|
||||
if node.id in visited:
|
||||
return
|
||||
visited.add(node.id)
|
||||
for stroke, (child, output) in node.edges.items():
|
||||
if output:
|
||||
outputs.add(output)
|
||||
visit(child, visited)
|
||||
if node.final and node.final_output:
|
||||
outputs.add(node.final_output)
|
||||
|
||||
visit(root)
|
||||
|
||||
sorted_outputs = sorted(outputs)
|
||||
|
||||
# Raw
|
||||
raw = b'\x00'.join(o.encode('utf-8') for o in sorted_outputs)
|
||||
raw_size = len(raw)
|
||||
|
||||
# Block compressed
|
||||
block_size = 4096
|
||||
blocks = []
|
||||
for i in range(0, len(raw), block_size):
|
||||
block = raw[i:i+block_size]
|
||||
blocks.append(zlib.compress(block, 9))
|
||||
compressed_size = sum(len(b) for b in blocks)
|
||||
index_size = len(blocks) * 4
|
||||
|
||||
# Full zlib
|
||||
full_zlib_size = len(zlib.compress(raw, 9))
|
||||
|
||||
return {
|
||||
'unique_count': len(sorted_outputs),
|
||||
'raw_size': raw_size,
|
||||
'block_compressed': compressed_size + index_size,
|
||||
'full_zlib': full_zlib_size,
|
||||
}
|
||||
|
||||
|
||||
# ─── Alternative: DAWG keys + implicit index + compressed values ───
|
||||
|
||||
def build_dawg_implicit_index(entries):
|
||||
"""
|
||||
Standard DAWG (no outputs on edges) but with implicit indexing.
|
||||
|
||||
DAWG traversal counts reachable final nodes → gives entry index.
|
||||
Entry index maps to value via simple array lookup.
|
||||
|
||||
Values stored as: entry_index → string_table_offset
|
||||
String table block-compressed.
|
||||
|
||||
Key difference from benchmark.py: here we build actual DAWG
|
||||
and measure REAL node count, then compute skip-count based size.
|
||||
"""
|
||||
DawgNode.next_id = 0
|
||||
root = DawgNode()
|
||||
unchecked = []
|
||||
minimized = {}
|
||||
prev_strokes = []
|
||||
|
||||
def minimize(down_to):
|
||||
for i in range(len(unchecked) - 1, down_to - 1, -1):
|
||||
parent, stroke, child = unchecked[i]
|
||||
sig = (child.final, tuple(sorted((k, v.id) for k, (v, _) in child.edges.items())))
|
||||
if sig in minimized:
|
||||
existing = minimized[sig]
|
||||
parent.edges[stroke] = (existing, "")
|
||||
else:
|
||||
minimized[sig] = child
|
||||
unchecked.pop()
|
||||
|
||||
for strokes, translation in entries:
|
||||
common = 0
|
||||
for i in range(min(len(strokes), len(prev_strokes))):
|
||||
if strokes[i] != prev_strokes[i]:
|
||||
break
|
||||
common += 1
|
||||
else:
|
||||
common = min(len(strokes), len(prev_strokes))
|
||||
|
||||
minimize(common)
|
||||
|
||||
if unchecked:
|
||||
node = unchecked[-1][2]
|
||||
else:
|
||||
node = root
|
||||
|
||||
for stroke in strokes[common:]:
|
||||
new_node = DawgNode()
|
||||
node.edges[stroke] = (new_node, "")
|
||||
unchecked.append((node, stroke, new_node))
|
||||
node = new_node
|
||||
|
||||
node.final = True
|
||||
prev_strokes = strokes
|
||||
|
||||
minimize(0)
|
||||
|
||||
# Count reachable finals for skip-count indexing
|
||||
def count_finals(node, cache=None):
|
||||
if cache is None:
|
||||
cache = {}
|
||||
if node.id in cache:
|
||||
return cache[node.id]
|
||||
c = 1 if node.final else 0
|
||||
for stroke in sorted(node.edges.keys()):
|
||||
child, _ = node.edges[stroke]
|
||||
c += count_finals(child, cache)
|
||||
cache[node.id] = c
|
||||
return c
|
||||
|
||||
count_finals(root)
|
||||
|
||||
return root
|
||||
|
||||
|
||||
def verify_implicit_index(root, entries):
|
||||
"""Verify skip-count indexing gives correct sequential indices."""
|
||||
|
||||
def lookup_index(node, strokes):
|
||||
"""Return the skip-count index for a stroke sequence."""
|
||||
idx = 0
|
||||
for stroke in strokes:
|
||||
# Count finals of all children with stroke < target
|
||||
for s in sorted(node.edges.keys()):
|
||||
if s == stroke:
|
||||
child, _ = node.edges[s]
|
||||
if child.final:
|
||||
# This child's final state comes before its children
|
||||
pass
|
||||
node = child
|
||||
if node.final:
|
||||
idx += 1 # count this final state
|
||||
break
|
||||
else:
|
||||
child, _ = node.edges[s]
|
||||
idx += count_subtree_finals(child)
|
||||
else:
|
||||
return -1
|
||||
return idx - 1 if node.final else -1
|
||||
|
||||
def count_subtree_finals(node, cache={}):
|
||||
if node.id in cache:
|
||||
return cache[node.id]
|
||||
c = 1 if node.final else 0
|
||||
for s in sorted(node.edges.keys()):
|
||||
child, _ = node.edges[s]
|
||||
c += count_subtree_finals(child, cache)
|
||||
cache[node.id] = c
|
||||
return c
|
||||
|
||||
# Verify first 100 entries get sequential indices
|
||||
correct = 0
|
||||
for expected_idx, (strokes, translation) in enumerate(entries[:100]):
|
||||
got_idx = lookup_index(root, strokes)
|
||||
if got_idx == expected_idx:
|
||||
correct += 1
|
||||
|
||||
return correct
|
||||
|
||||
|
||||
# ─── Main ───
|
||||
|
||||
def main():
|
||||
dict_path = '/tmp/plover-main.json'
|
||||
if not os.path.exists(dict_path):
|
||||
print("ERROR: Download Plover dict first:")
|
||||
print(" curl -sL 'https://raw.githubusercontent.com/openstenoproject/plover/main/plover/assets/main.json' -o /tmp/plover-main.json")
|
||||
sys.exit(1)
|
||||
|
||||
with open(dict_path) as f:
|
||||
raw_dict = json.load(f)
|
||||
|
||||
# Parse and sort
|
||||
parsed = []
|
||||
unique_stroke_vals = set()
|
||||
for stroke_str in sorted(raw_dict.keys()):
|
||||
strokes = tuple(parse_stroke(s) for s in stroke_str.split('/'))
|
||||
for s in strokes:
|
||||
unique_stroke_vals.add(s)
|
||||
parsed.append((strokes, raw_dict[stroke_str]))
|
||||
parsed.sort(key=lambda x: x[0])
|
||||
|
||||
n_entries = len(parsed)
|
||||
n_unique_strokes = len(unique_stroke_vals)
|
||||
translations = [v for _, v in parsed]
|
||||
|
||||
print(f"Entries: {n_entries}, Unique strokes: {n_unique_strokes}")
|
||||
print()
|
||||
|
||||
# ─── Approach A: DAWG-FST (outputs on edges) ───
|
||||
print("=" * 60)
|
||||
print("APPROACH A: DAWG-FST (outputs on edges)")
|
||||
print("=" * 60)
|
||||
|
||||
print("Building DAWG-FST...")
|
||||
root_fst = build_dawg_fst(parsed)
|
||||
|
||||
stats = measure_dawg_fst(root_fst)
|
||||
print(f" Nodes: {stats['nodes']}")
|
||||
print(f" Edges: {stats['edges']}")
|
||||
print(f" Unique output strings: {stats['unique_outputs']}")
|
||||
print(f" Total output bytes (on edges): {stats['total_output_bytes']}")
|
||||
print(f" Avg output length: {stats['avg_output_len']:.1f} bytes")
|
||||
|
||||
# Verify correctness
|
||||
print(" Verifying lookups...")
|
||||
correct, wrong, missing = verify_dawg_fst(root_fst, parsed[:1000])
|
||||
print(f" Verification (first 1000): {correct} correct, {wrong} wrong, {missing} missing")
|
||||
|
||||
# Output string table
|
||||
str_table = build_output_string_table(root_fst)
|
||||
print(f" Output string table:")
|
||||
print(f" Unique strings: {str_table['unique_count']}")
|
||||
print(f" Raw: {str_table['raw_size']/1024:.1f} KB")
|
||||
print(f" Block-compressed: {str_table['block_compressed']/1024:.1f} KB")
|
||||
print(f" Full zlib: {str_table['full_zlib']/1024:.1f} KB")
|
||||
|
||||
# Binary size estimate
|
||||
bin_est = estimate_binary_size(stats, n_unique_strokes)
|
||||
print(f" Binary encoding:")
|
||||
print(f" cbits={bin_est['cbits']} abits={bin_est['abits']} obits={bin_est['obits']}")
|
||||
print(f" Bits/edge: {bin_est['bits_per_edge']}")
|
||||
print(f" Structure: {bin_est['structure_bytes']/1024:.1f} KB")
|
||||
|
||||
total_a = bin_est['structure_bytes'] + str_table['block_compressed']
|
||||
print(f" TOTAL: {total_a/1024:.1f} KB")
|
||||
print()
|
||||
|
||||
# ─── Approach B: DAWG + implicit skip-count index ───
|
||||
print("=" * 60)
|
||||
print("APPROACH B: DAWG + skip-count index (no value array)")
|
||||
print("=" * 60)
|
||||
|
||||
print("Building standard DAWG...")
|
||||
root_std = build_dawg_implicit_index(parsed)
|
||||
|
||||
stats_std = measure_dawg_fst(root_std) # reuse measurement fn
|
||||
print(f" Nodes: {stats_std['nodes']}")
|
||||
print(f" Edges: {stats_std['edges']}")
|
||||
|
||||
# DAWG structure: edges need stroke + target + skip_count
|
||||
cbits = max(1, math.ceil(math.log2(max(n_unique_strokes, 2))))
|
||||
abits = max(1, math.ceil(math.log2(max(stats_std['nodes'], 2))))
|
||||
skipbits = max(1, math.ceil(math.log2(max(n_entries, 2))))
|
||||
|
||||
# Compact: last-edge flag saves storing child count
|
||||
bits_per_edge = cbits + abits + skipbits + 1 # stroke + target + skip + last_edge
|
||||
structure_bits = stats_std['edges'] * bits_per_edge + stats_std['nodes'] * 1 # is_final per node
|
||||
structure_bytes = (structure_bits + 7) // 8
|
||||
|
||||
print(f" cbits={cbits} abits={abits} skipbits={skipbits}")
|
||||
print(f" Bits/edge: {bits_per_edge}")
|
||||
print(f" Structure: {structure_bytes/1024:.1f} KB")
|
||||
|
||||
# Values: skip-count gives index → look up in ordered value array
|
||||
# Value array: translations in DAWG traversal order
|
||||
# Need: string table + offset array (index → string table position)
|
||||
unique_trans = sorted(set(translations))
|
||||
raw_strings = b'\x00'.join(t.encode('utf-8') for t in unique_trans)
|
||||
|
||||
# Dedup: entry → unique_string_id
|
||||
trans_to_id = {t: i for i, t in enumerate(unique_trans)}
|
||||
dedup_array = [trans_to_id[t] for t in translations]
|
||||
dedup_bits = math.ceil(math.log2(len(unique_trans)))
|
||||
dedup_bytes = (n_entries * dedup_bits + 7) // 8
|
||||
|
||||
# String table block-compressed
|
||||
blocks = []
|
||||
for i in range(0, len(raw_strings), 4096):
|
||||
blocks.append(zlib.compress(raw_strings[i:i+4096], 9))
|
||||
str_compressed = sum(len(b) for b in blocks) + len(blocks) * 4
|
||||
|
||||
total_b = structure_bytes + dedup_bytes + str_compressed
|
||||
print(f" Value dedup array: {dedup_bytes/1024:.1f} KB ({dedup_bits} bits/entry)")
|
||||
print(f" String table: {str_compressed/1024:.1f} KB")
|
||||
print(f" TOTAL: {total_b/1024:.1f} KB")
|
||||
print()
|
||||
|
||||
# ─── Approach C: DAWG + skip-count + NO dedup array ───
|
||||
# Instead of dedup array, store string offset directly in DAWG final nodes
|
||||
print("=" * 60)
|
||||
print("APPROACH C: DAWG + skip-count + direct string refs")
|
||||
print("=" * 60)
|
||||
|
||||
# Each final node stores a string table offset
|
||||
n_final = sum(1 for _ in range(1)) # need to count
|
||||
visited_c = set()
|
||||
n_final_c = 0
|
||||
def count_final(node):
|
||||
nonlocal n_final_c
|
||||
if node.id in visited_c:
|
||||
return
|
||||
visited_c.add(node.id)
|
||||
if node.final:
|
||||
n_final_c += 1
|
||||
for s, (child, _) in node.edges.items():
|
||||
count_final(child)
|
||||
count_final(root_std)
|
||||
|
||||
# But wait — with DAWG suffix sharing, multiple entries share final nodes
|
||||
# A shared final node can only store ONE string offset
|
||||
# This breaks dedup... unless we use the skip-count to disambiguate
|
||||
# Skip-count already gives unique index → use that as index into value array
|
||||
# So we STILL need the value array
|
||||
|
||||
# Alternative: don't share final nodes (partial DAWG — share internal only)
|
||||
# Then each final node = unique entry = unique string ref
|
||||
print(f" Final nodes (shared): {n_final_c}")
|
||||
print(f" Total entries: {n_entries}")
|
||||
print(f" Final nodes can't store unique refs with suffix sharing")
|
||||
print(f" → Must use skip-count index + value array (same as Approach B)")
|
||||
print()
|
||||
|
||||
# ─── Approach D: Partial DAWG (share internal only) + direct refs ───
|
||||
print("=" * 60)
|
||||
print("APPROACH D: Partial DAWG (no suffix sharing at finals)")
|
||||
print("=" * 60)
|
||||
|
||||
# Don't merge final nodes → each has unique string ref
|
||||
# Merge only internal nodes
|
||||
# Trade: more nodes but no value array needed
|
||||
|
||||
# In the standard DAWG we had 22K nodes.
|
||||
# Without suffix sharing at finals, estimate:
|
||||
# 147K entries = 147K unique final nodes + shared internal nodes
|
||||
# Internal nodes from DAWG: ~22K - final_shared ≈ much more nodes
|
||||
# This blows up the structure. Not good.
|
||||
print(f" Would need ~{n_entries} final nodes (no sharing)")
|
||||
print(f" Structure would be larger than value array savings")
|
||||
print(f" → Not viable")
|
||||
print()
|
||||
|
||||
# ─── Approach E: DAWG + varint value array ───
|
||||
print("=" * 60)
|
||||
print("APPROACH E: DAWG + varint-compressed value array")
|
||||
print("=" * 60)
|
||||
|
||||
# Instead of fixed-width dedup_bits per entry, use varint
|
||||
# Most translations are common (dedup IDs are small for frequent ones)
|
||||
# Sort unique translations by frequency → frequent = small ID → small varint
|
||||
|
||||
trans_freq = Counter(translations)
|
||||
sorted_by_freq = sorted(set(translations), key=lambda t: -trans_freq[t])
|
||||
freq_to_id = {t: i for i, t in enumerate(sorted_by_freq)}
|
||||
|
||||
# Varint encode: 7 bits per byte, high bit = continuation
|
||||
def varint_size(n):
|
||||
if n < 128: return 1
|
||||
if n < 16384: return 2
|
||||
if n < 2097152: return 3
|
||||
return 4
|
||||
|
||||
varint_total = sum(varint_size(freq_to_id[t]) for t in translations)
|
||||
|
||||
print(f" Fixed-width value array: {dedup_bytes/1024:.1f} KB")
|
||||
print(f" Varint value array: {varint_total/1024:.1f} KB")
|
||||
print(f" Savings: {(dedup_bytes - varint_total)/1024:.1f} KB")
|
||||
|
||||
total_e = structure_bytes + varint_total + str_compressed
|
||||
print(f" TOTAL: {total_e/1024:.1f} KB")
|
||||
print()
|
||||
|
||||
# ─── Approach F: Hybrid — DAWG keys + Huffman-coded values ───
|
||||
print("=" * 60)
|
||||
print("APPROACH F: DAWG + Huffman-coded value IDs")
|
||||
print("=" * 60)
|
||||
|
||||
# Huffman code the dedup IDs based on frequency
|
||||
# Theoretical minimum: entropy
|
||||
total_entries = len(translations)
|
||||
entropy_bits = 0
|
||||
for t, count in trans_freq.items():
|
||||
p = count / total_entries
|
||||
entropy_bits -= count * math.log2(p)
|
||||
entropy_bytes = int(entropy_bits / 8)
|
||||
|
||||
print(f" Entropy of value mapping: {entropy_bytes/1024:.1f} KB")
|
||||
print(f" (theoretical minimum for value array)")
|
||||
|
||||
total_f = structure_bytes + entropy_bytes + str_compressed
|
||||
print(f" DAWG structure: {structure_bytes/1024:.1f} KB")
|
||||
print(f" Huffman values: {entropy_bytes/1024:.1f} KB")
|
||||
print(f" String table: {str_compressed/1024:.1f} KB")
|
||||
print(f" TOTAL: {total_f/1024:.1f} KB")
|
||||
print()
|
||||
|
||||
# ─── Summary ───
|
||||
print("=" * 60)
|
||||
print("SUMMARY")
|
||||
print("=" * 60)
|
||||
print(f" Target: 300 KB")
|
||||
print()
|
||||
|
||||
results = [
|
||||
("A: DAWG-FST (edge outputs)", total_a),
|
||||
("B: DAWG + skip + dedup array", total_b),
|
||||
("E: DAWG + varint values", total_e),
|
||||
("F: DAWG + Huffman values", total_f),
|
||||
]
|
||||
results.sort(key=lambda x: x[1])
|
||||
|
||||
for name, size in results:
|
||||
kb = size / 1024
|
||||
marker = " ✓ FITS!" if kb <= 300 else f" ({kb-300:+.0f} KB over)"
|
||||
print(f" {name:40s}: {kb:8.1f} KB{marker}")
|
||||
|
||||
# Breakdown of best approach
|
||||
print()
|
||||
best_name, best_size = results[0]
|
||||
print(f" Best: {best_name}")
|
||||
print(f" Breakdown:")
|
||||
if "FST" in best_name:
|
||||
print(f" DAWG-FST structure: {bin_est['structure_bytes']/1024:.1f} KB")
|
||||
print(f" Output string table: {str_table['block_compressed']/1024:.1f} KB")
|
||||
elif "Huffman" in best_name:
|
||||
print(f" DAWG structure: {structure_bytes/1024:.1f} KB")
|
||||
print(f" Huffman value IDs: {entropy_bytes/1024:.1f} KB")
|
||||
print(f" String table: {str_compressed/1024:.1f} KB")
|
||||
elif "varint" in best_name:
|
||||
print(f" DAWG structure: {structure_bytes/1024:.1f} KB")
|
||||
print(f" Varint value array: {varint_total/1024:.1f} KB")
|
||||
print(f" String table: {str_compressed/1024:.1f} KB")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue