hb_store_lmdb.erl - LMDB Persistent Key-Value Store

Overview

Purpose: High-performance embedded transactional database storage using LMDB
Module: hb_store_lmdb
Pattern: Singleton server with asynchronous writes and automatic link resolution

This module provides a persistent key-value store backend using LMDB (Lightning Memory Database), implementing the HyperBEAM store interface. Each database environment gets its own dedicated server process to manage transactions and coordinate writes with a dual-flush strategy for optimal performance.

Dependencies

External: elmdb (Erlang LMDB bindings)
HyperBEAM: hb_util, hb_store
Erlang/OTP: filelib, filename, persistent_term
Records: #tx{} from include/hb.hrl

Public Functions Overview

%% Lifecycle Management
-spec start(StoreOpts) -> {ok, Instance} | {error, Reason}.
-spec stop(StoreOpts) -> ok.
-spec reset(StoreOpts) -> ok.
-spec scope() -> local.
-spec scope(StoreOpts) -> local.
 
%% Core Store Operations
-spec read(Opts, Path) -> {ok, Value} | not_found | {error, Reason}.
-spec write(Opts, Path, Value) -> ok | retry.
-spec list(Opts, Path) -> {ok, [Key]} | {error, Reason}.
-spec type(Opts, Key) -> composite | simple | not_found.
 
%% Hierarchical Structure
-spec make_group(Opts, Path) -> ok.
-spec make_link(Opts, Existing, New) -> ok.
 
%% Path Operations
-spec path(Opts, PathParts) -> binary().
-spec add_path(Opts, Path1, Path2) -> binary().
-spec resolve(Opts, Path) -> binary().

Public Functions

1. start/1

-spec start(StoreOpts) -> {ok, Instance} | {error, Reason}
    when
        StoreOpts :: #{ <<"name">> := binary(), <<"capacity">> => integer() },
        Instance :: #{ <<"env">> => reference(), <<"db">> => reference() },
        Reason :: term().

Description: Initialize or connect to an LMDB database instance. Uses singleton pattern - multiple calls with same configuration return same server. Creates directory if needed and opens LMDB environment with specified size limit.

Configuration:

<<"name">>: Database directory path (required)
<<"capacity">>: Maximum database size in bytes (default: 16GB)

Test Code:

-module(test_hb_store_lmdb).
-include_lib("eunit/include/eunit.hrl").
-include("include/hb.hrl").
 
%% Helper to generate unique store names (avoids LMDB already_open errors)
unique_store(Prefix) ->
    Id = integer_to_binary(erlang:unique_integer([positive])),
    #{
        <<"name">> => <<"/tmp/lmdb-test-", Prefix/binary, "-", Id/binary>>,
        <<"store-module">> => hb_store_lmdb
    }.
 
start_basic_test() ->
    Store = maps:merge(unique_store(<<"start">>), #{<<"capacity">> => 1024 * 1024 * 10}),
    {ok, Instance} = hb_store_lmdb:start(Store),
    ?assert(is_map(Instance)),
    ?assert(maps:is_key(<<"env">>, Instance)),
    ?assert(maps:is_key(<<"db">>, Instance)).
 
start_with_defaults_test() ->
    Store = unique_store(<<"defaults">>),
    {ok, _Instance} = hb_store_lmdb:start(Store).
 
start_invalid_opts_test() ->
    ?assertEqual(
        {error, {badarg, <<"StoreOpts must be a map">>}},
        hb_store_lmdb:start(not_a_map)
    ).

2. stop/1

-spec stop(StoreOpts) -> ok
    when
        StoreOpts :: map().

Description: Stop the LMDB storage system and close the database environment. Cleans up resources including persistent_term storage.

Test Code:

stop_test() ->
    Store = unique_store(<<"stop">>),
    {ok, _} = hb_store_lmdb:start(Store),
    ?assertEqual(ok, hb_store_lmdb:stop(Store)).

3. reset/1

-spec reset(StoreOpts) -> ok
    when
        StoreOpts :: map().

Description: Clear all data from the database by deleting the database directory (rm -Rf). Typically called at the START of tests to ensure a clean state from previous test runs.

Test Code:

reset_clears_directory_test() ->
    Dir = <<"/tmp/lmdb-reset-test-dir">>,
    Store = #{<<"name">> => Dir, <<"store-module">> => hb_store_lmdb},
    %% Ensure clean state
    hb_store_lmdb:reset(Store),
    %% Write data (creates directory)
    hb_store_lmdb:write(Store, <<"key">>, <<"value">>),
    ?assert(filelib:is_dir(binary_to_list(Dir))),
    %% Stop then reset
    hb_store_lmdb:stop(Store),
    hb_store_lmdb:reset(Store),
    %% Directory should be gone
    ?assertNot(filelib:is_dir(binary_to_list(Dir))).

4. scope/0, scope/1

-spec scope() -> local.
-spec scope(StoreOpts) -> local
    when
        StoreOpts :: map().

Description: Return the scope of the store. LMDB is always local storage.

Test Code:

scope_test() ->
    ?assertEqual(local, hb_store_lmdb:scope()),
    ?assertEqual(local, hb_store_lmdb:scope(#{})).

5. write/3

-spec write(Opts, Path, Value) -> ok | retry
    when
        Opts :: map(),
        Path :: binary() | list(),
        Value :: binary().

Description: Write a key-value pair to the database asynchronously. Accepts either binary path or list of path segments. Returns immediately - writes are accumulated and flushed periodically. Note: write handles database startup internally.

Test Code:

write_basic_test() ->
    Store = unique_store(<<"write">>),
    ?assertEqual(ok, hb_store_lmdb:write(Store, <<"key">>, <<"value">>)),
    ?assertEqual({ok, <<"value">>}, hb_store_lmdb:read(Store, <<"key">>)).
 
write_with_path_list_test() ->
    Store = unique_store(<<"writelist">>),
    ?assertEqual(ok, hb_store_lmdb:write(Store, [<<"dir">>, <<"key">>], <<"value">>)),
    ?assertEqual({ok, <<"value">>}, hb_store_lmdb:read(Store, <<"dir/key">>)).

6. read/2

-spec read(Opts, Path) -> {ok, Value} | not_found | {error, Reason}
    when
        Opts :: map(),
        Path :: binary() | list(),
        Value :: binary(),
        Reason :: term().

Description: Read a value from the database with automatic link resolution. Accepts binary path or list of segments. If key stores a link (value starting with "link:"), automatically follows it to the target.

Test Code:

read_not_found_test() ->
    Store = unique_store(<<"readnf">>),
    ?assertEqual(not_found, hb_store_lmdb:read(Store, <<"nonexistent">>)).
 
read_with_link_test() ->
    Store = unique_store(<<"readlink">>),
    hb_store_lmdb:write(Store, <<"target">>, <<"data">>),
    hb_store_lmdb:make_link(Store, <<"target">>, <<"link">>),
    ?assertEqual({ok, <<"data">>}, hb_store_lmdb:read(Store, <<"link">>)).

7. list/2

-spec list(Opts, Path) -> {ok, [Key]} | {error, Reason}
    when
        Opts :: map(),
        Path :: binary(),
        Key :: binary(),
        Reason :: term().

Description: List all immediate children of a group path. Returns only direct children, not nested descendants. Automatically resolves links.

Test Code:

list_basic_test() ->
    Store = unique_store(<<"list">>),
    hb_store_lmdb:make_group(Store, <<"group">>),
    hb_store_lmdb:write(Store, <<"group/child1">>, <<"val1">>),
    hb_store_lmdb:write(Store, <<"group/child2">>, <<"val2">>),
    {ok, Children} = hb_store_lmdb:list(Store, <<"group">>),
    ?assertEqual([<<"child1">>, <<"child2">>], lists:sort(Children)).
 
list_empty_group_test() ->
    Store = unique_store(<<"listempty">>),
    hb_store_lmdb:make_group(Store, <<"empty">>),
    ?assertEqual({ok, []}, hb_store_lmdb:list(Store, <<"empty">>)).

8. type/2

-spec type(Opts, Key) -> composite | simple | not_found
    when
        Opts :: map(),
        Key :: binary().

Description: Determine whether a key represents a group (composite) or simple value. Keys storing the literal binary "group" are composite. Automatically follows links to check target type.

Test Code:

type_simple_test() ->
    Store = unique_store(<<"typesimple">>),
    hb_store_lmdb:write(Store, <<"key">>, <<"value">>),
    ?assertEqual(simple, hb_store_lmdb:type(Store, <<"key">>)).
 
type_composite_test() ->
    Store = unique_store(<<"typecomp">>),
    hb_store_lmdb:make_group(Store, <<"group">>),
    ?assertEqual(composite, hb_store_lmdb:type(Store, <<"group">>)).
 
type_not_found_test() ->
    Store = unique_store(<<"typenf">>),
    ?assertEqual(not_found, hb_store_lmdb:type(Store, <<"nonexistent">>)).

9. make_group/2

-spec make_group(Opts, Path) -> ok
    when
        Opts :: map(),
        Path :: binary().

Description: Create a group (directory-like structure) at the specified path. Groups store the literal binary "group" as their value and can contain child entries.

Test Code:

make_group_test() ->
    Store = unique_store(<<"mkgroup">>),
    ?assertEqual(ok, hb_store_lmdb:make_group(Store, <<"mygroup">>)),
    ?assertEqual(composite, hb_store_lmdb:type(Store, <<"mygroup">>)).
 
make_nested_group_test() ->
    Store = unique_store(<<"mknested">>),
    hb_store_lmdb:make_group(Store, <<"parent">>),
    hb_store_lmdb:make_group(Store, <<"parent/child">>),
    ?assertEqual(composite, hb_store_lmdb:type(Store, <<"parent/child">>)).

10. make_link/3

-spec make_link(Opts, Existing, New) -> ok
    when
        Opts :: map(),
        Existing :: binary(),
        New :: binary().

Description: Create a symbolic link from New to Existing. The New key will store "link:" prefix followed by Existing path. Reading New will automatically resolve to Existing's value.

Test Code:

make_link_test() ->
    Store = unique_store(<<"mklink">>),
    hb_store_lmdb:write(Store, <<"target">>, <<"data">>),
    ?assertEqual(ok, hb_store_lmdb:make_link(Store, <<"target">>, <<"link">>)),
    ?assertEqual({ok, <<"data">>}, hb_store_lmdb:read(Store, <<"link">>)).
 
make_link_preserves_type_test() ->
    Store = unique_store(<<"mklinktype">>),
    hb_store_lmdb:make_group(Store, <<"target-group">>),
    hb_store_lmdb:make_link(Store, <<"target-group">>, <<"link-group">>),
    ?assertEqual(composite, hb_store_lmdb:type(Store, <<"link-group">>)).

11. path/2

-spec path(Opts, PathParts) -> binary()
    when
        Opts :: map(),
        PathParts :: list().

Description: Convert a list of path segments into a binary path string, joined with "/".

Test Code:

path_test() ->
    ?assertEqual(<<"a/b/c">>, hb_store_lmdb:path(#{}, [<<"a">>, <<"b">>, <<"c">>])).
 
path_single_test() ->
    ?assertEqual(<<"key">>, hb_store_lmdb:path(#{}, <<"key">>)).

12. add_path/3

-spec add_path(Opts, Path1, Path2) -> binary()
    when
        Opts :: map(),
        Path1 :: binary(),
        Path2 :: binary().

Description: Join two paths together with "/" separator.

Test Code:

add_path_test() ->
    ?assertEqual(<<"base/child">>, hb_store_lmdb:add_path(#{}, <<"base">>, <<"child">>)).

13. resolve/2

-spec resolve(Opts, Path) -> binary()
    when
        Opts :: map(),
        Path :: binary() | list().

Description: Resolve a path by following all links in the path segments (except the final segment). Returns the fully resolved path.

Test Code:

resolve_direct_test() ->
    Store = unique_store(<<"resdir">>),
    hb_store_lmdb:write(Store, <<"direct">>, <<"value">>),
    ?assertEqual(<<"direct">>, hb_store_lmdb:resolve(Store, <<"direct">>)).
 
resolve_with_link_test() ->
    Store = unique_store(<<"reslink">>),
    %% resolve only follows links in intermediate path segments
    hb_store_lmdb:make_group(Store, <<"target">>),
    hb_store_lmdb:write(Store, <<"target/key">>, <<"value">>),
    hb_store_lmdb:make_link(Store, <<"target">>, <<"link">>),
    ?assertEqual(<<"target/key">>, hb_store_lmdb:resolve(Store, [<<"link">>, <<"key">>])).

Common Patterns

%% Initialize database
StoreOpts = #{
    <<"name">> => <<"cache-mainnet/lmdb">>,
    <<"store-module">> => hb_store_lmdb,
    <<"capacity">> => 16 * 1024 * 1024 * 1024  % 16GB
},
{ok, _Instance} = hb_store_lmdb:start(StoreOpts).
 
%% Basic write and read
hb_store_lmdb:write(StoreOpts, <<"key">>, <<"value">>),
{ok, Value} = hb_store_lmdb:read(StoreOpts, <<"key">>).
 
%% Hierarchical data with groups
hb_store_lmdb:make_group(StoreOpts, <<"messages">>),
hb_store_lmdb:write(StoreOpts, <<"messages/msg1">>, <<"data1">>),
{ok, Children} = hb_store_lmdb:list(StoreOpts, <<"messages">>).
 
%% Symbolic links for data deduplication
hb_store_lmdb:write(StoreOpts, <<"data/hash123">>, <<"actual_data">>),
hb_store_lmdb:make_link(StoreOpts, <<"data/hash123">>, <<"messages/msg1/body">>).
 
%% Cleanup
hb_store_lmdb:stop(StoreOpts).

Storage Architecture

Value Types

Value Pattern	Type	Meaning
`<<"group">>`	Composite	Directory/group containing children
`<<"link:", Path/binary>>`	Link	Symbolic link to another path
Any other binary	Simple	Regular data value

Configuration Constants

-define(DEFAULT_SIZE, 16 * 1024 * 1024 * 1024).  % 16GB default capacity
-define(MAX_REDIRECTS, 1000).                    % Max link chain depth
-define(MAX_PENDING_WRITES, 400).                % Force flush threshold

Notes

Singleton Pattern: Each database path gets one server process
Automatic Startup: Data operations handle startup internally - do not call start before write/read
Link Chain Limit: Maximum 1000 redirects to prevent infinite loops
Path Format: Always uses "/" as separator