True split-dict: partition dictionary across both halves
Importance-based partitioning — highest-importance entries on left (central) for zero-latency local lookup, remainder on right (peripheral) queried over BLE on miss. Both halves embed their own MPHF binary. Configurable block size for tighter compression.
This commit is contained in:
parent
9d2d5e4e2d
commit
20218fa2ab
7 changed files with 209 additions and 58 deletions
|
|
@ -18,6 +18,7 @@
|
|||
|
||||
#if IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
|
||||
#include "split_dict.h"
|
||||
#include "dict_mphf.h"
|
||||
#elif IS_ENABLED(CONFIG_STENO_DICT_MPHF)
|
||||
#include "dict_mphf.h"
|
||||
#else
|
||||
|
|
@ -26,12 +27,10 @@
|
|||
|
||||
LOG_MODULE_DECLARE(zmk, CONFIG_ZMK_LOG_LEVEL);
|
||||
|
||||
#if !IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
|
||||
extern const uint8_t _steno_dict_start[];
|
||||
extern const uint8_t _steno_dict_end[];
|
||||
#endif
|
||||
|
||||
#if IS_ENABLED(CONFIG_STENO_DICT_MPHF) && !IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
|
||||
#if IS_ENABLED(CONFIG_STENO_DICT_MPHF) || IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
|
||||
static struct dict_mphf mphf_dict;
|
||||
#endif
|
||||
|
||||
|
|
@ -57,6 +56,12 @@ static void multi_timeout_handler(struct k_work *work);
|
|||
static const char *do_lookup(const uint32_t *strokes, uint8_t count)
|
||||
{
|
||||
#if IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
|
||||
/* Try local partition first (zero latency) */
|
||||
const char *local = dict_mphf_lookup(&mphf_dict, strokes, count);
|
||||
if (local) {
|
||||
return local;
|
||||
}
|
||||
/* Miss → query remote partition over BLE */
|
||||
static char split_buf[128];
|
||||
int ret = split_dict_lookup(strokes, count, split_buf, sizeof(split_buf));
|
||||
return (ret > 0) ? split_buf : NULL;
|
||||
|
|
@ -70,6 +75,11 @@ static const char *do_lookup(const uint32_t *strokes, uint8_t count)
|
|||
static bool do_has_prefix(const uint32_t *strokes, uint8_t count)
|
||||
{
|
||||
#if IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
|
||||
/* Check local prefix table first */
|
||||
if (count == 1 && dict_mphf_has_prefix(&mphf_dict, strokes[0])) {
|
||||
return true;
|
||||
}
|
||||
/* Fall through to remote */
|
||||
return split_dict_has_prefix(strokes, count);
|
||||
#elif IS_ENABLED(CONFIG_STENO_DICT_MPHF)
|
||||
return (count == 1) ? dict_mphf_has_prefix(&mphf_dict, strokes[0]) : false;
|
||||
|
|
@ -225,6 +235,21 @@ static int behavior_steno_init(const struct device *dev)
|
|||
k_work_init_delayable(&state.multi_timeout, multi_timeout_handler);
|
||||
|
||||
#if IS_ENABLED(CONFIG_STENO_SPLIT_DICT)
|
||||
/* Init local partition dict */
|
||||
{
|
||||
size_t dict_size = _steno_dict_end - _steno_dict_start;
|
||||
if (dict_size > 4) {
|
||||
int ret = dict_mphf_init(&mphf_dict, _steno_dict_start, dict_size);
|
||||
if (ret == 0) {
|
||||
LOG_INF("Local partition loaded (%u bytes)", (unsigned)dict_size);
|
||||
} else {
|
||||
LOG_ERR("Local partition init failed: %d", ret);
|
||||
}
|
||||
} else {
|
||||
LOG_WRN("No local partition embedded");
|
||||
}
|
||||
}
|
||||
/* Init BLE client for remote partition */
|
||||
split_dict_init();
|
||||
dict_ready = true;
|
||||
#else
|
||||
|
|
|
|||
|
|
@ -188,6 +188,9 @@ int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len)
|
|||
dict->blocks_start = dict->str_data_start;
|
||||
}
|
||||
|
||||
/* Block size from header (0 = legacy default 4096) */
|
||||
dict->blk_size = hdr->block_size ? hdr->block_size : DICT_MPHF_BLOCK_SIZE;
|
||||
|
||||
/* Prefix table at end */
|
||||
uint32_t prefix_bytes = (uint32_t)hdr->prefix_count * 4;
|
||||
if (len >= prefix_bytes) {
|
||||
|
|
@ -213,8 +216,8 @@ static const char *resolve_string(const struct dict_mphf *dict, uint32_t val_id)
|
|||
}
|
||||
|
||||
/* Block-compressed: decompress the right block */
|
||||
uint32_t block_idx = str_offset / DICT_MPHF_BLOCK_SIZE;
|
||||
uint32_t in_block_off = str_offset % DICT_MPHF_BLOCK_SIZE;
|
||||
uint32_t block_idx = str_offset / dict->blk_size;
|
||||
uint32_t in_block_off = str_offset % dict->blk_size;
|
||||
|
||||
if (block_idx >= dict->block_count) {
|
||||
return NULL;
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ struct dict_mphf_header {
|
|||
uint8_t value_bits;
|
||||
uint8_t disp_bits;
|
||||
uint16_t prefix_count;
|
||||
uint32_t reserved0;
|
||||
uint32_t block_size; /* zlib block size (0 = default 4096) */
|
||||
uint32_t reserved1;
|
||||
} __attribute__((packed));
|
||||
|
||||
|
|
@ -52,6 +52,7 @@ struct dict_mphf {
|
|||
const uint32_t *block_dir; /* block offset directory */
|
||||
const uint8_t *blocks_start; /* start of compressed blocks */
|
||||
uint32_t str_data_len; /* total string data section length */
|
||||
uint32_t blk_size; /* actual block size from header */
|
||||
};
|
||||
|
||||
int dict_mphf_init(struct dict_mphf *dict, const void *data, size_t len);
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@
|
|||
|
||||
#include "split_dict.h"
|
||||
#include "split_cache.h"
|
||||
#include "dict_mphf.h"
|
||||
|
||||
LOG_MODULE_REGISTER(split_dict, CONFIG_STENO_SPLIT_LOG_LEVEL);
|
||||
|
||||
|
|
@ -33,10 +34,11 @@ static uint8_t seq_counter;
|
|||
/* Cache instance */
|
||||
static struct split_cache dict_cache;
|
||||
|
||||
/* External trie lookup (peripheral side) */
|
||||
extern int trie_lookup(const uint32_t *strokes, uint8_t count,
|
||||
char *result, size_t result_size);
|
||||
extern bool trie_has_prefix(const uint32_t *strokes, uint8_t count);
|
||||
/* Local MPHF dict for peripheral-side GATT lookups */
|
||||
extern const uint8_t _steno_dict_start[];
|
||||
extern const uint8_t _steno_dict_end[];
|
||||
static struct dict_mphf peripheral_mphf;
|
||||
static bool peripheral_dict_ready;
|
||||
|
||||
/* --- Helpers --- */
|
||||
|
||||
|
|
@ -91,14 +93,17 @@ static ssize_t dict_query_write_cb(struct bt_conn *conn,
|
|||
resp->msg_type = STENO_MSG_RESPONSE;
|
||||
resp->seq = pkt->seq;
|
||||
|
||||
char translation[128];
|
||||
int ret = trie_lookup(strokes, stroke_count, translation, sizeof(translation));
|
||||
const char *translation = NULL;
|
||||
if (peripheral_dict_ready) {
|
||||
translation = dict_mphf_lookup(&peripheral_mphf, strokes, stroke_count);
|
||||
}
|
||||
|
||||
if (ret > 0) {
|
||||
if (translation) {
|
||||
uint16_t tlen = (uint16_t)strlen(translation);
|
||||
resp->status = STENO_STATUS_FOUND;
|
||||
resp->data_len = (uint16_t)ret;
|
||||
memcpy(resp->data, translation, ret);
|
||||
response_len = sizeof(struct steno_response_pkt) + ret;
|
||||
resp->data_len = tlen;
|
||||
memcpy(resp->data, translation, tlen);
|
||||
response_len = sizeof(struct steno_response_pkt) + tlen;
|
||||
} else {
|
||||
resp->status = STENO_STATUS_NOT_FOUND;
|
||||
resp->data_len = 0;
|
||||
|
|
@ -135,7 +140,9 @@ static ssize_t dict_prefix_write_cb(struct bt_conn *conn,
|
|||
resp->seq = pkt->seq;
|
||||
resp->data_len = 0;
|
||||
|
||||
if (trie_has_prefix(strokes, stroke_count)) {
|
||||
if (peripheral_dict_ready &&
|
||||
stroke_count == 1 &&
|
||||
dict_mphf_has_prefix(&peripheral_mphf, strokes[0])) {
|
||||
resp->status = STENO_STATUS_PREFIX_ONLY;
|
||||
} else {
|
||||
resp->status = STENO_STATUS_NOT_FOUND;
|
||||
|
|
@ -188,14 +195,17 @@ static ssize_t dict_batch_write_cb(struct bt_conn *conn,
|
|||
resp->msg_type = STENO_MSG_RESPONSE;
|
||||
resp->seq = pkt->seq;
|
||||
|
||||
char translation[128];
|
||||
int ret = trie_lookup(strokes, stroke_count, translation, sizeof(translation));
|
||||
const char *translation = NULL;
|
||||
if (peripheral_dict_ready) {
|
||||
translation = dict_mphf_lookup(&peripheral_mphf, strokes, stroke_count);
|
||||
}
|
||||
|
||||
if (ret > 0) {
|
||||
if (translation) {
|
||||
uint16_t tlen = (uint16_t)strlen(translation);
|
||||
resp->status = STENO_STATUS_FOUND;
|
||||
resp->data_len = (uint16_t)ret;
|
||||
memcpy(resp->data, translation, ret);
|
||||
response_len = sizeof(struct steno_response_pkt) + ret;
|
||||
resp->data_len = tlen;
|
||||
memcpy(resp->data, translation, tlen);
|
||||
response_len = sizeof(struct steno_response_pkt) + tlen;
|
||||
} else {
|
||||
resp->status = STENO_STATUS_NOT_FOUND;
|
||||
resp->data_len = 0;
|
||||
|
|
@ -453,6 +463,18 @@ int split_dict_init(void)
|
|||
seq_counter = 0;
|
||||
split_conn = NULL;
|
||||
|
||||
/* Init peripheral-side MPHF dict for GATT lookups */
|
||||
size_t dict_size = _steno_dict_end - _steno_dict_start;
|
||||
if (dict_size > 4) {
|
||||
int ret = dict_mphf_init(&peripheral_mphf, _steno_dict_start, dict_size);
|
||||
if (ret == 0) {
|
||||
peripheral_dict_ready = true;
|
||||
LOG_INF("Peripheral partition loaded (%u bytes)", (unsigned)dict_size);
|
||||
} else {
|
||||
LOG_ERR("Peripheral partition init failed: %d", ret);
|
||||
}
|
||||
}
|
||||
|
||||
LOG_INF("Split dict initialized");
|
||||
return 0;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue