True split-dict: partition dictionary across both halves
Importance-based partitioning — highest-importance entries on left (central) for zero-latency local lookup, remainder on right (peripheral) queried over BLE on miss. Both halves embed their own MPHF binary. Configurable block size for tighter compression.
This commit is contained in:
parent
9d2d5e4e2d
commit
20218fa2ab
7 changed files with 209 additions and 58 deletions
|
|
@ -9,30 +9,31 @@ target_include_directories(app PRIVATE
|
|||
)
|
||||
|
||||
# ── Split-dict mode ──────────────────────────────────
|
||||
# Central: behavior engine + BLE dict client
|
||||
# Peripheral: dict embed + lookup engine + GATT server
|
||||
# TRUE split: both halves have local MPHF partition + BLE fallback
|
||||
# Central: behavior engine + local left partition + BLE client
|
||||
# Peripheral: local right partition + GATT server
|
||||
if(CONFIG_STENO_SPLIT_DICT)
|
||||
|
||||
if(CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
|
||||
# Central side: behavior engine queries peripheral over BLE
|
||||
# Central: behavior engine + local dict + BLE client for remote partition
|
||||
target_sources(app PRIVATE
|
||||
src/behavior_steno.c
|
||||
src/output.c
|
||||
src/formatter.c
|
||||
src/undo.c
|
||||
src/dict_embed.S
|
||||
src/dict_mphf.c
|
||||
src/split_dict.c
|
||||
src/split_cache.c
|
||||
)
|
||||
else()
|
||||
# Peripheral side: dict embedded here, serves GATT queries
|
||||
# Peripheral: local dict + GATT server for central queries
|
||||
target_sources(app PRIVATE
|
||||
src/dict_embed.S
|
||||
src/dict_mphf.c
|
||||
src/split_dict.c
|
||||
src/split_cache.c
|
||||
)
|
||||
if(CONFIG_STENO_DICT_MPHF)
|
||||
target_sources(app PRIVATE src/dict_mphf.c)
|
||||
else()
|
||||
target_sources(app PRIVATE src/trie.c)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# ── Non-split mode ───────────────────────────────────
|
||||
|
|
@ -55,12 +56,11 @@ else()
|
|||
endif()
|
||||
|
||||
# ── Dictionary compilation ───────────────────────────
|
||||
# Build dict binary when we embed it (non-split, or split peripheral)
|
||||
# Both halves need dict embed in split mode; non-split same as before
|
||||
set(STENO_NEED_DICT_EMBED FALSE)
|
||||
if(CONFIG_STENO_SPLIT_DICT)
|
||||
if(NOT CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
|
||||
set(STENO_NEED_DICT_EMBED TRUE)
|
||||
endif()
|
||||
# Both central and peripheral need their own partition
|
||||
set(STENO_NEED_DICT_EMBED TRUE)
|
||||
elseif(NOT CONFIG_ZMK_SPLIT OR CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
|
||||
set(STENO_NEED_DICT_EMBED TRUE)
|
||||
endif()
|
||||
|
|
@ -96,19 +96,43 @@ if(STENO_NEED_DICT_EMBED)
|
|||
|
||||
# Compile dict binary
|
||||
if(EXISTS ${STENO_DICT_SRC})
|
||||
if(CONFIG_STENO_DICT_MPHF)
|
||||
# Fetch at build time if hash changed
|
||||
if(DEFINED STENO_DICT_NAME)
|
||||
add_custom_command(
|
||||
OUTPUT ${STENO_DICT_SRC}.stamp
|
||||
COMMAND ${Python3_EXECUTABLE} ${STENO_FETCH}
|
||||
${STENO_DICT_NAME} ${STENO_DICTS_DIR}
|
||||
COMMAND ${CMAKE_COMMAND} -E touch ${STENO_DICT_SRC}.stamp
|
||||
COMMENT "Checking ${STENO_DICT_NAME} dictionary for updates"
|
||||
)
|
||||
add_custom_target(steno_dict_fetch DEPENDS ${STENO_DICT_SRC}.stamp)
|
||||
# Fetch at build time if hash changed
|
||||
if(DEFINED STENO_DICT_NAME)
|
||||
add_custom_command(
|
||||
OUTPUT ${STENO_DICT_SRC}.stamp
|
||||
COMMAND ${Python3_EXECUTABLE} ${STENO_FETCH}
|
||||
${STENO_DICT_NAME} ${STENO_DICTS_DIR}
|
||||
COMMAND ${CMAKE_COMMAND} -E touch ${STENO_DICT_SRC}.stamp
|
||||
COMMENT "Checking ${STENO_DICT_NAME} dictionary for updates"
|
||||
)
|
||||
add_custom_target(steno_dict_fetch DEPENDS ${STENO_DICT_SRC}.stamp)
|
||||
endif()
|
||||
|
||||
if(CONFIG_STENO_SPLIT_DICT)
|
||||
# Split mode: compile the appropriate partition
|
||||
if(CONFIG_ZMK_SPLIT_ROLE_CENTRAL)
|
||||
set(STENO_SPLIT_PART "left")
|
||||
set(STENO_SPLIT_SIZE ${CONFIG_STENO_DICT_LEFT_MAX_SIZE})
|
||||
else()
|
||||
set(STENO_SPLIT_PART "right")
|
||||
set(STENO_SPLIT_SIZE ${CONFIG_STENO_DICT_RIGHT_MAX_SIZE})
|
||||
endif()
|
||||
|
||||
add_custom_command(
|
||||
OUTPUT ${STENO_DICT_BIN}
|
||||
COMMAND ${Python3_EXECUTABLE}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_mphf.py
|
||||
${STENO_DICT_SRC} ${STENO_DICT_BIN}
|
||||
--split-part ${STENO_SPLIT_PART}
|
||||
--left-size ${CONFIG_STENO_DICT_LEFT_MAX_SIZE}
|
||||
--right-size ${CONFIG_STENO_DICT_RIGHT_MAX_SIZE}
|
||||
--max-size ${STENO_SPLIT_SIZE}
|
||||
--block-size 2048
|
||||
DEPENDS ${STENO_DICT_SRC}
|
||||
${CMAKE_CURRENT_SOURCE_DIR}/tools/compile_mphf.py
|
||||
COMMENT "Compiling steno dictionary (${STENO_SPLIT_PART} partition)"
|
||||
)
|
||||
elseif(CONFIG_STENO_DICT_MPHF)
|
||||
add_custom_command(
|
||||
OUTPUT ${STENO_DICT_BIN}
|
||||
COMMAND ${Python3_EXECUTABLE}
|
||||
|
|
|
|||
28
Kconfig
28
Kconfig
|
|
@ -81,15 +81,31 @@ config STENO_DICT_MAX_SIZE
|
|||
The MPHF compiler auto-trims to fit.
|
||||
|
||||
menuconfig STENO_SPLIT_DICT
|
||||
bool "Split dictionary storage on peripheral"
|
||||
bool "Split dictionary across both halves"
|
||||
default n
|
||||
select STENO_DICT_MPHF
|
||||
help
|
||||
Store the steno dictionary on the peripheral half and
|
||||
perform lookups over BLE. This frees flash on the central
|
||||
side at the cost of added lookup latency.
|
||||
Partition the steno dictionary across both keyboard halves.
|
||||
Left (central) gets highest-importance entries for zero-latency
|
||||
local lookup. Right (peripheral) gets remaining entries, queried
|
||||
over BLE on local miss. Both halves have their own MPHF dict.
|
||||
|
||||
if STENO_SPLIT_DICT
|
||||
|
||||
config STENO_DICT_LEFT_MAX_SIZE
|
||||
int "Left half dict budget (bytes)"
|
||||
default 430080
|
||||
help
|
||||
Flash budget for left (central) partition. Default 420KB.
|
||||
Highest-importance entries fill this first.
|
||||
|
||||
config STENO_DICT_RIGHT_MAX_SIZE
|
||||
int "Right half dict budget (bytes)"
|
||||
default 545792
|
||||
help
|
||||
Flash budget for right (peripheral) partition. Default 533KB.
|
||||
Remaining entries after left partition is filled.
|
||||
|
||||
config STENO_SPLIT_CACHE_SIZE
|
||||
int "LRU cache entries on central side"
|
||||
default 64
|
||||
|
|
@ -104,6 +120,10 @@ config STENO_SPLIT_TIMEOUT_MS
|
|||
default 50
|
||||
range 10 500
|
||||
|
||||
config STENO_SPLIT_LOG_LEVEL
|
||||
int "Split dict log level"
|
||||
default 3
|
||||
|
||||
endif # STENO_SPLIT_DICT
|
||||
|
||||
endif # STENO_ENGINE
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@
|
|||
|
||||
#if IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
|
||||
#include "split_dict.h"
|
||||
#include "dict_mphf.h"
|
||||
#elif IS_ENABLED(CONFIG_STENO_DICT_MPHF)
|
||||
#include "dict_mphf.h"
|
||||
#else
|
||||
|
|
@ -26,12 +27,10 @@
|
|||
|
||||
LOG_MODULE_DECLARE(zmk, CONFIG_ZMK_LOG_LEVEL);
|
||||
|
||||
#if !IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
|
||||
extern const uint8_t _steno_dict_start[];
|
||||
extern const uint8_t _steno_dict_end[];
|
||||
#endif
|
||||
|
||||
#if IS_ENABLED(CONFIG_STENO_DICT_MPHF) && !IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
|
||||
#if IS_ENABLED(CONFIG_STENO_DICT_MPHF) || IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
|
||||
static struct dict_mphf mphf_dict;
|
||||
#endif
|
||||
|
||||
|
|
@ -57,6 +56,12 @@ static void multi_timeout_handler(struct k_work *work);
|
|||
static const char *do_lookup(const uint32_t *strokes, uint8_t count)
|
||||
{
|
||||
#if IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
|
||||
/* Try local partition first (zero latency) */
|
||||
const char *local = dict_mphf_lookup(&mphf_dict, strokes, count);
|
||||
if (local) {
|
||||
return local;
|
||||
}
|
||||
/* Miss → query remote partition over BLE */
|
||||
static char split_buf[128];
|
||||
int ret = split_dict_lookup(strokes, count, split_buf, sizeof(split_buf));
|
||||
return (ret > 0) ? split_buf : NULL;
|
||||
|
|
@ -70,6 +75,11 @@ static const char *do_lookup(const uint32_t *strokes, uint8_t count)
|
|||
static bool do_has_prefix(const uint32_t *strokes, uint8_t count)
|
||||
{
|
||||
#if IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
|
||||
/* Check local prefix table first */
|
||||
if (count == 1 && dict_mphf_has_prefix(&mphf_dict, strokes[0])) {
|
||||
return true;
|
||||
}
|
||||
/* Fall through to remote */
|
||||
return split_dict_has_prefix(strokes, count);
|
||||
#elif IS_ENABLED(CONFIG_STENO_DICT_MPHF)
|
||||
return (count == 1) ? dict_mphf_has_prefix(&mphf_dict, strokes[0]) : false;
|
||||
|
|
@ -225,6 +235,21 @@ static int behavior_steno_init(const struct device *dev)
|
|||
k_work_init_delayable(&state.multi_timeout, multi_timeout_handler);
|
||||
|
||||
#if IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
|
||||
/* Init local partition dict */
|
||||
{
|
||||
size_t dict_size = _steno_dict_end - _steno_dict_start;
|
||||
if (dict_size > 4) {
|
||||
int ret = dict_mphf_init(&mphf_dict, _steno_dict_start, dict_size);
|
||||
if (ret == 0) {
|
||||
LOG_INF("Local partition loaded (%u bytes)", (unsigned)dict_size);
|
||||
} else {
|
||||
LOG_ERR("Local partition init failed: %d", ret);
|
||||
}
|
||||
} else {
|
||||
LOG_WRN("No local partition embedded");
|
||||
}
|
||||
}
|
||||
/* Init BLE client for remote partition */
|
||||
split_dict_init();
|
||||
dict_ready = true;
|
||||
#else
|
||||
|
|
|
|||
|
|
@ -188,6 +188,9 @@ int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len)
|
|||
dict->blocks_start = dict->str_data_start;
|
||||
}
|
||||
|
||||
/* Block size from header (0 = legacy default 4096) */
|
||||
dict->blk_size = hdr->block_size ? hdr->block_size : DICT_MPHF_BLOCK_SIZE;
|
||||
|
||||
/* Prefix table at end */
|
||||
uint32_t prefix_bytes = (uint32_t)hdr->prefix_count * 4;
|
||||
if (len >= prefix_bytes) {
|
||||
|
|
@ -213,8 +216,8 @@ static const char *resolve_string(const struct dict_mphf *dict, uint32_t val_id)
|
|||
}
|
||||
|
||||
/* Block-compressed: decompress the right block */
|
||||
uint32_t block_idx = str_offset / DICT_MPHF_BLOCK_SIZE;
|
||||
uint32_t in_block_off = str_offset % DICT_MPHF_BLOCK_SIZE;
|
||||
uint32_t block_idx = str_offset / dict->blk_size;
|
||||
uint32_t in_block_off = str_offset % dict->blk_size;
|
||||
|
||||
if (block_idx >= dict->block_count) {
|
||||
return NULL;
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ struct dict_mphf_header {
|
|||
uint8_t value_bits;
|
||||
uint8_t disp_bits;
|
||||
uint16_t prefix_count;
|
||||
uint32_t reserved0;
|
||||
uint32_t block_size; /* zlib block size (0 = default 4096) */
|
||||
uint32_t reserved1;
|
||||
} __attribute__((packed));
|
||||
|
||||
|
|
@ -52,6 +52,7 @@ struct dict_mphf {
|
|||
const uint32_t *block_dir; /* block offset directory */
|
||||
const uint8_t *blocks_start; /* start of compressed blocks */
|
||||
uint32_t str_data_len; /* total string data section length */
|
||||
uint32_t blk_size; /* actual block size from header */
|
||||
};
|
||||
|
||||
int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len);
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@
|
|||
|
||||
#include "split_dict.h"
|
||||
#include "split_cache.h"
|
||||
#include "dict_mphf.h"
|
||||
|
||||
LOG_MODULE_REGISTER(split_dict, CONFIG_STENO_SPLIT_LOG_LEVEL);
|
||||
|
||||
|
|
@ -33,10 +34,11 @@ static uint8_t seq_counter;
|
|||
/* Cache instance */
|
||||
static struct split_cache dict_cache;
|
||||
|
||||
/* External trie lookup (peripheral side) */
|
||||
extern int trie_lookup(const uint32_t *strokes, uint8_t count,
|
||||
char *result, size_t result_size);
|
||||
extern bool trie_has_prefix(const uint32_t *strokes, uint8_t count);
|
||||
/* Local MPHF dict for peripheral-side GATT lookups */
|
||||
extern const uint8_t _steno_dict_start[];
|
||||
extern const uint8_t _steno_dict_end[];
|
||||
static struct dict_mphf peripheral_mphf;
|
||||
static bool peripheral_dict_ready;
|
||||
|
||||
/* --- Helpers --- */
|
||||
|
||||
|
|
@ -91,14 +93,17 @@ static ssize_t dict_query_write_cb(struct bt_conn *conn,
|
|||
resp->msg_type = STENO_MSG_RESPONSE;
|
||||
resp->seq = pkt->seq;
|
||||
|
||||
char translation[128];
|
||||
int ret = trie_lookup(strokes, stroke_count, translation, sizeof(translation));
|
||||
const char *translation = NULL;
|
||||
if (peripheral_dict_ready) {
|
||||
translation = dict_mphf_lookup(&peripheral_mphf, strokes, stroke_count);
|
||||
}
|
||||
|
||||
if (ret > 0) {
|
||||
if (translation) {
|
||||
uint16_t tlen = (uint16_t)strlen(translation);
|
||||
resp->status = STENO_STATUS_FOUND;
|
||||
resp->data_len = (uint16_t)ret;
|
||||
memcpy(resp->data, translation, ret);
|
||||
response_len = sizeof(struct steno_response_pkt) + ret;
|
||||
resp->data_len = tlen;
|
||||
memcpy(resp->data, translation, tlen);
|
||||
response_len = sizeof(struct steno_response_pkt) + tlen;
|
||||
} else {
|
||||
resp->status = STENO_STATUS_NOT_FOUND;
|
||||
resp->data_len = 0;
|
||||
|
|
@ -135,7 +140,9 @@ static ssize_t dict_prefix_write_cb(struct bt_conn *conn,
|
|||
resp->seq = pkt->seq;
|
||||
resp->data_len = 0;
|
||||
|
||||
if (trie_has_prefix(strokes, stroke_count)) {
|
||||
if (peripheral_dict_ready &&
|
||||
stroke_count == 1 &&
|
||||
dict_mphf_has_prefix(&peripheral_mphf, strokes[0])) {
|
||||
resp->status = STENO_STATUS_PREFIX_ONLY;
|
||||
} else {
|
||||
resp->status = STENO_STATUS_NOT_FOUND;
|
||||
|
|
@ -188,14 +195,17 @@ static ssize_t dict_batch_write_cb(struct bt_conn *conn,
|
|||
resp->msg_type = STENO_MSG_RESPONSE;
|
||||
resp->seq = pkt->seq;
|
||||
|
||||
char translation[128];
|
||||
int ret = trie_lookup(strokes, stroke_count, translation, sizeof(translation));
|
||||
const char *translation = NULL;
|
||||
if (peripheral_dict_ready) {
|
||||
translation = dict_mphf_lookup(&peripheral_mphf, strokes, stroke_count);
|
||||
}
|
||||
|
||||
if (ret > 0) {
|
||||
if (translation) {
|
||||
uint16_t tlen = (uint16_t)strlen(translation);
|
||||
resp->status = STENO_STATUS_FOUND;
|
||||
resp->data_len = (uint16_t)ret;
|
||||
memcpy(resp->data, translation, ret);
|
||||
response_len = sizeof(struct steno_response_pkt) + ret;
|
||||
resp->data_len = tlen;
|
||||
memcpy(resp->data, translation, tlen);
|
||||
response_len = sizeof(struct steno_response_pkt) + tlen;
|
||||
} else {
|
||||
resp->status = STENO_STATUS_NOT_FOUND;
|
||||
resp->data_len = 0;
|
||||
|
|
@ -453,6 +463,18 @@ int split_dict_init(void)
|
|||
seq_counter = 0;
|
||||
split_conn = NULL;
|
||||
|
||||
/* Init peripheral-side MPHF dict for GATT lookups */
|
||||
size_t dict_size = _steno_dict_end - _steno_dict_start;
|
||||
if (dict_size > 4) {
|
||||
int ret = dict_mphf_init(&peripheral_mphf, _steno_dict_start, dict_size);
|
||||
if (ret == 0) {
|
||||
peripheral_dict_ready = true;
|
||||
LOG_INF("Peripheral partition loaded (%u bytes)", (unsigned)dict_size);
|
||||
} else {
|
||||
LOG_ERR("Peripheral partition init failed: %d", ret);
|
||||
}
|
||||
}
|
||||
|
||||
LOG_INF("Split dict initialized");
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -230,10 +230,11 @@ def build_chd(keys_and_bytes, entry_count):
|
|||
|
||||
# ─── Compilation ───
|
||||
|
||||
def compile_mphf(entries, max_size=None):
|
||||
def compile_mphf(entries, max_size=None, block_size=4096):
|
||||
"""
|
||||
entries: list of (stroke_str, translation) from JSON dict
|
||||
max_size: max output size in bytes (default: 462*1024 = 473088)
|
||||
block_size: zlib compression block size (smaller = better ratio, more blocks)
|
||||
|
||||
Returns: bytes (the compiled binary) or None if can't fit
|
||||
"""
|
||||
|
|
@ -286,7 +287,6 @@ def compile_mphf(entries, max_size=None):
|
|||
string_offsets.append(len(string_data_raw))
|
||||
string_data_raw += t.encode('utf-8') + b'\x00'
|
||||
|
||||
block_size = 4096
|
||||
compressed_blocks = []
|
||||
for i in range(0, len(string_data_raw), block_size):
|
||||
block = string_data_raw[i:i + block_size]
|
||||
|
|
@ -388,7 +388,7 @@ def compile_mphf(entries, max_size=None):
|
|||
# magic: u32, version: u16, flags: u16,
|
||||
# entry_count: u32, bucket_count: u32, unique_count: u32,
|
||||
# value_bits: u8, disp_bits: u8, prefix_count: u16,
|
||||
# reserved0: u32, reserved1: u32
|
||||
# block_size: u32, reserved1: u32
|
||||
header = struct.pack('<IHHIIIBBHii',
|
||||
0x4F4E5453, # magic "STNO"
|
||||
2, # version
|
||||
|
|
@ -399,7 +399,7 @@ def compile_mphf(entries, max_size=None):
|
|||
value_bits, # value_bits
|
||||
disp_bits, # disp_bits
|
||||
prefix_count, # prefix_count
|
||||
0, # reserved0
|
||||
block_size, # block_size (was reserved0)
|
||||
0, # reserved1
|
||||
)
|
||||
assert len(header) == 32, f"Header is {len(header)} bytes, expected 32"
|
||||
|
|
@ -475,6 +475,7 @@ def compile_mphf(entries, max_size=None):
|
|||
'disp_bits': disp_bits,
|
||||
'max_displacement': max_disp,
|
||||
'prefix_count': prefix_count,
|
||||
'block_size': block_size,
|
||||
'disp_section_bytes': len(disp_section),
|
||||
'val_section_bytes': len(val_section),
|
||||
'fp_section_bytes': len(fp_section),
|
||||
|
|
@ -488,8 +489,8 @@ def compile_mphf(entries, max_size=None):
|
|||
|
||||
def print_stats(stats):
|
||||
"""Print size breakdown statistics."""
|
||||
total = stats['entry_count']
|
||||
print(f"Entries: {stats['entry_count']}")
|
||||
print(f"Block size: {stats.get('block_size', 4096)} bytes")
|
||||
print(f"MPHF displacements: {stats['disp_section_bytes']/1024:.1f} KB "
|
||||
f"({stats['bucket_count']} buckets, {stats['disp_bits']} bits each)")
|
||||
print(f"Value array: {stats['val_section_bytes']/1024:.1f} KB "
|
||||
|
|
@ -504,6 +505,32 @@ def print_stats(stats):
|
|||
print(f"Total: {stats['total_bytes']/1024:.1f} KB")
|
||||
|
||||
|
||||
def partition_entries(entries, left_budget, right_budget):
|
||||
"""Partition dict entries by importance into left (central) and right (peripheral).
|
||||
|
||||
Left gets highest-importance entries first (most common single-stroke words).
|
||||
Right gets remaining entries. Both are trimmed to their flash budgets via
|
||||
compile_mphf's internal trimming.
|
||||
|
||||
Returns (left_entries, right_entries).
|
||||
"""
|
||||
sorted_entries = sorted(entries, key=lambda e: score_entry(e[0], e[1]))
|
||||
|
||||
# Rough estimate: ~15 bytes per entry average (conservative)
|
||||
# compile_mphf will trim further if needed
|
||||
est_bytes_per_entry = 15
|
||||
left_max = left_budget // est_bytes_per_entry
|
||||
right_max = right_budget // est_bytes_per_entry
|
||||
|
||||
# Cap to actual count
|
||||
left_max = min(left_max, len(sorted_entries))
|
||||
|
||||
left_entries = sorted_entries[:left_max]
|
||||
right_entries = sorted_entries[left_max:]
|
||||
|
||||
return left_entries, right_entries
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description='Compile steno dictionary to MPHF binary format')
|
||||
parser.add_argument('input', help='Input JSON dictionary (Plover format)')
|
||||
|
|
@ -516,6 +543,14 @@ def main():
|
|||
help='Print size breakdown statistics')
|
||||
parser.add_argument('--verify', action='store_true', default=True,
|
||||
help='Verify compiled dict (default: true)')
|
||||
parser.add_argument('--split-part', choices=['left', 'right'],
|
||||
help='Build one partition of a split dict (left=central, right=peripheral)')
|
||||
parser.add_argument('--left-size', type=int, default=430080,
|
||||
help='Left partition flash budget in bytes (default: 430080 = 420KB)')
|
||||
parser.add_argument('--right-size', type=int, default=545792,
|
||||
help='Right partition flash budget in bytes (default: 545792 = 533KB)')
|
||||
parser.add_argument('--block-size', type=int, default=4096,
|
||||
help='Zlib compression block size (default: 4096, try 2048/1024 for tighter packing)')
|
||||
args = parser.parse_args()
|
||||
|
||||
# Load dictionary
|
||||
|
|
@ -531,7 +566,26 @@ def main():
|
|||
entries = entries_scored[:args.max_entries]
|
||||
print(f"Trimmed to {len(entries)} entries (--max-entries)", file=sys.stderr)
|
||||
|
||||
result = compile_mphf(entries, max_size=args.max_size)
|
||||
# Split-partition mode
|
||||
if args.split_part:
|
||||
left_entries, right_entries = partition_entries(
|
||||
entries, args.left_size, args.right_size)
|
||||
|
||||
if args.split_part == 'left':
|
||||
print(f"Split partition: LEFT (central) — {len(left_entries)} entries, "
|
||||
f"budget {args.left_size} bytes", file=sys.stderr)
|
||||
entries = left_entries
|
||||
max_size = args.left_size
|
||||
else:
|
||||
print(f"Split partition: RIGHT (peripheral) — {len(right_entries)} entries, "
|
||||
f"budget {args.right_size} bytes", file=sys.stderr)
|
||||
entries = right_entries
|
||||
max_size = args.right_size
|
||||
else:
|
||||
max_size = args.max_size
|
||||
|
||||
result = compile_mphf(entries, max_size=max_size,
|
||||
block_size=args.block_size)
|
||||
|
||||
if result is None:
|
||||
print("Compilation failed", file=sys.stderr)
|
||||
|
|
@ -544,8 +598,10 @@ def main():
|
|||
|
||||
print(f"Wrote {len(binary)} bytes to {args.output}", file=sys.stderr)
|
||||
|
||||
if args.stats:
|
||||
if args.stats or args.split_part:
|
||||
print()
|
||||
if args.split_part:
|
||||
print(f"=== {args.split_part.upper()} partition ===")
|
||||
print_stats(stats)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue