hb_store.erl - Storage Abstraction Layer

Overview

Purpose: Abstract key-value store operations across multiple storage backends
Module: hb_store
Pattern: Behavior-based pluggable storage with access control and fallback chains

This module provides a unified interface for storage operations, allowing node operators to use different storage backends (filesystem, LMDB, remote, etc.) interchangeably. Supports store chains for fallback, access control policies, and scope-based filtering.

Dependencies

HyperBEAM: hb_opts, hb_maps, hb_util, hb_path
Erlang/OTP: persistent_term, timer
Records: Store-specific implementations

Public Functions Overview

%% Behavior Definition
-spec behavior_info(callbacks) -> [{atom(), arity()}].
 
%% Lifecycle (Section 1)
-spec start(Stores) -> ok.
-spec stop(Stores) -> ok | not_found.
-spec reset(Stores) -> ok | not_found.
 
%% Instance Management (Section 2)
-spec find(StoreOpts) -> Instance | ok.
 
%% Data Operations (Section 3)
-spec read(Stores, Key) -> {ok, Value} | not_found.
-spec write(Stores, Key, Value) -> ok | not_found.
 
%% Query Operations (Section 4)
-spec type(Stores, Key) -> simple | composite | not_found.
-spec list(Stores, Key) -> {ok, [Key]} | not_found.
-spec match(Stores, Pattern) -> {ok, [Key]} | not_found.
 
%% Group and Link Operations (Section 5)
-spec make_group(Stores, Key) -> ok | not_found.
-spec make_link(Stores, Existing, New) -> ok | not_found.
-spec resolve(Stores, Key) -> ResolvedKey | not_found.
 
%% Store Filtering (Section 6)
-spec filter(Stores, FilterFun) -> FilteredStores.
-spec scope(OptsOrStores, Scope) -> FilteredStores.
 
%% Store Ordering (Section 7)
-spec sort(Stores, PreferenceOrder | ScoreMap) -> SortedStores.
 
%% Path Operations (Section 8)
-spec path(Key) -> NormalizedPath.
-spec path(Stores, Key) -> NormalizedPath.
-spec add_path(Path1, Path2) -> JoinedPath.
-spec add_path(Stores, Path1, Path2) -> JoinedPath.
-spec join(PathParts) -> JoinedPath.
 
%% Testing Utilities (Section 9)
-spec test_stores() -> [StoreOpts].
-spec generate_test_suite(Suite) -> EUnitTests.
-spec generate_test_suite(Suite, Stores) -> EUnitTests.

Public Functions

1. start/1, stop/1, reset/1

-spec start(Stores) -> ok
    when
        Stores :: map() | [map()].
 
-spec stop(Stores) -> ok | not_found
    when
        Stores :: map() | [map()].
 
-spec reset(Stores) -> ok | not_found
    when
        Stores :: map() | [map()].

Description: Lifecycle management for stores. start/1 initializes stores (calls find/1 internally), stop/1 shuts them down, reset/1 clears all data. All functions accept single store or list of stores.

Test Code:

-module(test_hb_store_lifecycle).
-include_lib("eunit/include/eunit.hrl").
 
start_single_store_test() ->
    Store = #{
        <<"store-module">> => hb_store_fs,
        <<"name">> => <<"lifecycle-start-single">>
    },
    ?assertEqual(ok, hb_store:start(Store)).
 
start_multiple_stores_test() ->
    Stores = [
        #{<<"store-module">> => hb_store_fs, <<"name">> => <<"lifecycle-multi-1">>},
        #{<<"store-module">> => hb_store_fs, <<"name">> => <<"lifecycle-multi-2">>}
    ],
    ?assertEqual(ok, hb_store:start(Stores)).
 
stop_store_test() ->
    Store = #{
        <<"store-module">> => hb_store_fs,
        <<"name">> => <<"lifecycle-stop">>
    },
    hb_store:start(Store),
    %% stop delegates to store module's stop callback
    Result = hb_store:stop(Store),
    ?assert(Result =:= ok orelse Result =:= not_found).
 
stop_multiple_stores_test() ->
    Stores = [
        #{<<"store-module">> => hb_store_fs, <<"name">> => <<"lifecycle-stop-1">>},
        #{<<"store-module">> => hb_store_fs, <<"name">> => <<"lifecycle-stop-2">>}
    ],
    hb_store:start(Stores),
    Result = hb_store:stop(Stores),
    ?assert(Result =:= ok orelse Result =:= not_found).
 
reset_clears_data_test() ->
    Store = #{
        <<"store-module">> => hb_store_fs,
        <<"name">> => <<"lifecycle-reset">>
    },
    hb_store:start(Store),
    hb_store:write(Store, <<"key">>, <<"value">>),
    ?assertMatch({ok, _}, hb_store:read(Store, <<"key">>)),
    hb_store:reset(Store),
    hb_store:start(Store),
    ?assertEqual(not_found, hb_store:read(Store, <<"key">>)).
 
full_lifecycle_test() ->
    Store = #{
        <<"store-module">> => hb_store_fs,
        <<"name">> => <<"lifecycle-full">>
    },
    %% Start
    ?assertEqual(ok, hb_store:start(Store)),
    %% Use
    ?assertEqual(ok, hb_store:write(Store, <<"k">>, <<"v">>)),
    ?assertEqual({ok, <<"v">>}, hb_store:read(Store, <<"k">>)),
    %% Reset
    hb_store:reset(Store),
    %% Stop
    hb_store:stop(Store).

2. find/1

-spec find(StoreOpts) -> Instance | ok
    when
        StoreOpts :: #{<<"store-module">> := module(), binary() => term()},
        Instance :: map().

Description: Find or spawn a store instance by its options. Checks process dictionary first, then persistent_term, then spawns a new instance if needed. Store instances are cached for performance. If a store process has died, it will be respawned automatically.

Test Code:

-module(test_hb_store_find).
-include_lib("eunit/include/eunit.hrl").
 
find_creates_instance_test() ->
    Store = #{
        <<"store-module">> => hb_store_fs,
        <<"name">> => <<"find-test-", (integer_to_binary(rand:uniform(10000)))/binary>>
    },
    %% First call spawns instance
    Result1 = hb_store:find(Store),
    ?assert(Result1 =:= ok orelse is_map(Result1)),
    %% Second call returns cached instance
    Result2 = hb_store:find(Store),
    ?assert(Result2 =:= ok orelse is_map(Result2)).
 
find_caches_in_process_dict_test() ->
    Store = #{
        <<"store-module">> => hb_store_fs,
        <<"name">> => <<"find-cache-", (integer_to_binary(rand:uniform(10000)))/binary>>
    },
    %% First call spawns and caches
    _Result1 = hb_store:find(Store),
    %% Second call should use process dictionary cache
    Result2 = hb_store:find(Store),
    ?assert(Result2 =:= ok orelse is_map(Result2)).

3. read/2, write/3

-spec read(Stores, Key) -> {ok, Value} | not_found
    when
        Stores :: map() | [map()],
        Key :: binary() | list(),
        Value :: binary() | map().
 
-spec write(Stores, Key, Value) -> ok | not_found
    when
        Stores :: map() | [map()],
        Key :: binary() | list(),
        Value :: binary() | map().

Description: Read and write operations. Tries each store in the chain until one succeeds. Respects access control policies. Keys can be binary or list (for hierarchical paths).

Test Code:

-module(test_hb_store_rw).
-include_lib("eunit/include/eunit.hrl").
 
read_write_binary_key_test() ->
    Store = hb_test_utils:test_store(hb_store_fs),
    Key = <<"test-key">>,
    Value = <<"test-value">>,
    
    ?assertEqual(ok, hb_store:write(Store, Key, Value)),
    ?assertEqual({ok, Value}, hb_store:read(Store, Key)).
 
read_write_hierarchical_key_test() ->
    Store = hb_test_utils:test_store(hb_store_fs),
    hb_store:make_group(Store, <<"parent">>),
    Key = [<<"parent">>, <<"child">>],
    Value = <<"nested-value">>,
    
    ?assertEqual(ok, hb_store:write(Store, Key, Value)),
    ?assertEqual({ok, Value}, hb_store:read(Store, Key)).
 
read_not_found_test() ->
    Store = hb_test_utils:test_store(hb_store_fs),
    hb_store:start(Store),
    ?assertEqual(not_found, hb_store:read(Store, <<"nonexistent">>)).
 
write_overwrite_test() ->
    Store = hb_test_utils:test_store(hb_store_fs),
    Key = <<"overwrite-key">>,
    ?assertEqual(ok, hb_store:write(Store, Key, <<"first">>)),
    ?assertEqual(ok, hb_store:write(Store, Key, <<"second">>)),
    ?assertEqual({ok, <<"second">>}, hb_store:read(Store, Key)).
 
read_from_chain_test() ->
    Store1 = hb_test_utils:test_store(hb_store_fs),
    Store2 = #{
        <<"store-module">> => hb_store_fs,
        <<"name">> => <<"chain-store2">>
    },
    hb_store:start(Store2),
    hb_store:write(Store2, <<"key">>, <<"in-store2">>),
    
    %% Chain tries Store1 first (not found), then Store2
    Chain = [Store1, Store2],
    ?assertEqual({ok, <<"in-store2">>}, hb_store:read(Chain, <<"key">>)).
 
write_fallback_chain_test() ->
    ReadOnlyStore = #{
        <<"store-module">> => hb_store_fs,
        <<"name">> => <<"ro-chain">>,
        <<"access">> => [<<"read">>]
    },
    WriteStore = hb_test_utils:test_store(hb_store_fs),
    hb_store:start(ReadOnlyStore),
    Chain = [ReadOnlyStore, WriteStore],
    
    %% Write falls through to WriteStore (ReadOnlyStore denies write)
    ?assertEqual(ok, hb_store:write(Chain, <<"key">>, <<"val">>)),
    %% Value only in WriteStore
    ?assertEqual(not_found, hb_store:read([ReadOnlyStore], <<"key">>)),
    ?assertMatch({ok, _}, hb_store:read([WriteStore], <<"key">>)).

4. type/2, list/2, match/2

-spec type(Stores, Key) -> simple | composite | not_found
    when
        Stores :: map() | [map()],
        Key :: binary() | list().
 
-spec list(Stores, Key) -> {ok, [Key]} | not_found
    when
        Stores :: map() | [map()],
        Key :: binary() | list().
 
-spec match(Stores, Pattern) -> {ok, [Key]} | not_found
    when
        Stores :: map() | [map()],
        Pattern :: map(),
        Key :: binary().

Description: type/2 returns whether a key contains a simple value or composite (group). list/2 lists keys in a composite group (use only in debugging - slow for most stores). match/2 finds keys matching a pattern of key-value pairs.

Test Code:

-module(test_hb_store_query).
-include_lib("eunit/include/eunit.hrl").
 
type_simple_test() ->
    Store = hb_test_utils:test_store(hb_store_fs),
    hb_store:write(Store, <<"key">>, <<"value">>),
    ?assertEqual(simple, hb_store:type(Store, <<"key">>)).
 
type_composite_test() ->
    Store = hb_test_utils:test_store(hb_store_fs),
    hb_store:make_group(Store, <<"group">>),
    hb_store:write(Store, [<<"group">>, <<"item">>], <<"value">>),
    ?assertEqual(composite, hb_store:type(Store, <<"group">>)).
 
type_not_found_test() ->
    Store = hb_test_utils:test_store(hb_store_fs),
    hb_store:start(Store),
    ?assertEqual(not_found, hb_store:type(Store, <<"nonexistent">>)).
 
list_group_test() ->
    Store = hb_test_utils:test_store(hb_store_fs),
    Group = <<"test-group">>,
    hb_store:make_group(Store, Group),
    hb_store:write(Store, [Group, <<"item1">>], <<"val1">>),
    hb_store:write(Store, [Group, <<"item2">>], <<"val2">>),
    
    {ok, Items} = hb_store:list(Store, Group),
    ?assertEqual(2, length(Items)),
    ?assert(lists:member(<<"item1">>, Items)),
    ?assert(lists:member(<<"item2">>, Items)).
 
match_not_found_test() ->
    Store = hb_test_utils:test_store(hb_store_fs),
    hb_store:start(Store),
    %% match returns not_found when no keys match pattern
    Result = hb_store:match(Store, #{<<"nonexistent">> => <<"value">>}),
    ?assertEqual(not_found, Result).

5. make_group/2, make_link/3, resolve/2

-spec make_group(Stores, Key) -> ok | not_found
    when
        Stores :: map() | [map()],
        Key :: binary() | list().
 
-spec make_link(Stores, Existing, New) -> ok | not_found
    when
        Stores :: map() | [map()],
        Existing :: binary() | list(),
        New :: binary() | list().
 
-spec resolve(Stores, Key) -> {ok, ResolvedKey} | not_found
    when
        Stores :: map() | [map()],
        Key :: binary() | list(),
        ResolvedKey :: binary() | list().

Description: make_group/2 creates a composite key (directory/group). make_link/3 creates a symbolic link from New to Existing. resolve/2 follows links through the store to find the ultimate target path.

Test Code:

-module(test_hb_store_group_link).
-include_lib("eunit/include/eunit.hrl").
 
make_group_test() ->
    Store = hb_test_utils:test_store(hb_store_fs),
    ?assertEqual(ok, hb_store:make_group(Store, <<"mygroup">>)),
    ?assertEqual(composite, hb_store:type(Store, <<"mygroup">>)).
 
make_link_test() ->
    Store = hb_test_utils:test_store(hb_store_fs),
    Target = <<"target-key">>,
    Link = <<"link-key">>,
    Value = <<"linked-value">>,
    
    hb_store:write(Store, Target, Value),
    hb_store:make_link(Store, Target, Link),
    
    {ok, ReadValue} = hb_store:read(Store, Link),
    ?assertEqual(Value, ReadValue).
 
%% From hb_store.erl: simple_path_resolution_test
simple_link_resolution_test() ->
    Store = hb_test_utils:test_store(hb_store_fs),
    hb_store:write(Store, <<"test-file">>, <<"test-data">>),
    hb_store:make_link(Store, <<"test-file">>, <<"test-link">>),
    ?assertEqual({ok, <<"test-data">>}, hb_store:read(Store, <<"test-link">>)).
 
%% From hb_store.erl: resursive_path_resolution_test
recursive_link_resolution_test() ->
    Store = hb_test_utils:test_store(hb_store_fs),
    hb_store:write(Store, <<"test-file">>, <<"test-data">>),
    hb_store:make_link(Store, <<"test-file">>, <<"test-link">>),
    hb_store:make_link(Store, <<"test-link">>, <<"test-link2">>),
    ?assertEqual({ok, <<"test-data">>}, hb_store:read(Store, <<"test-link2">>)).
 
%% From hb_store.erl: hierarchical_path_resolution_test
hierarchical_link_resolution_test() ->
    Store = hb_test_utils:test_store(hb_store_fs),
    hb_store:make_group(Store, <<"test-dir1">>),
    hb_store:write(Store, [<<"test-dir1">>, <<"test-file">>], <<"test-data">>),
    hb_store:make_link(Store, [<<"test-dir1">>], <<"test-link">>),
    ?assertEqual(
        {ok, <<"test-data">>},
        hb_store:read(Store, [<<"test-link">>, <<"test-file">>])
    ).

6. scope/2, filter/2

-spec scope(OptsOrStores, Scope) -> FilteredStores
    when
        OptsOrStores :: map() | [map()],
        Scope :: atom() | [atom()],
        FilteredStores :: [map()].
 
-spec filter(Stores, FilterFun) -> FilteredStores
    when
        Stores :: map() | [map()],
        FilterFun :: function(),
        FilteredStores :: [map()].

Description: Filter stores by scope or custom criteria. scope/2 filters by predefined scopes (in_memory, local, remote, arweave). When passed a map, returns map with filtered store key. When passed a list, returns filtered list. filter/2 uses a custom function fun(Scope, Store) -> boolean(). Default scope is local.

Test Code:

-module(test_hb_store_filter).
-include_lib("eunit/include/eunit.hrl").
 
scope_with_list_input_test() ->
    LocalStore = #{
        <<"store-module">> => hb_store_fs,
        <<"name">> => <<"scope-local">>,
        <<"scope">> => local
    },
    RemoteStore = #{
        <<"store-module">> => hb_store_gateway,
        <<"scope">> => remote
    },
    Stores = [LocalStore, RemoteStore],
    
    %% When passed a list, returns filtered list
    LocalOnly = hb_store:scope(Stores, local),
    ?assertEqual(1, length(LocalOnly)),
    ?assertEqual(hb_store_fs, maps:get(<<"store-module">>, hd(LocalOnly))).
 
scope_with_map_input_test() ->
    LocalStore = #{
        <<"store-module">> => hb_store_fs,
        <<"name">> => <<"scope-map-local">>,
        <<"scope">> => local
    },
    RemoteStore = #{
        <<"store-module">> => hb_store_gateway,
        <<"scope">> => remote
    },
    Stores = [LocalStore, RemoteStore],
    
    %% When passed a map with store key, returns map with filtered store
    Result = hb_store:scope(#{store => Stores}, local),
    ?assert(is_map(Result)),
    FilteredStores = maps:get(store, Result),
    ?assertEqual(1, length(FilteredStores)).
 
scope_multiple_scopes_test() ->
    MemStore = #{<<"store-module">> => hb_store_fs, <<"scope">> => in_memory},
    LocalStore = #{<<"store-module">> => hb_store_fs, <<"scope">> => local},
    RemoteStore = #{<<"store-module">> => hb_store_gateway, <<"scope">> => remote},
    Stores = [MemStore, LocalStore, RemoteStore],
    
    %% Filter to local and in_memory only
    Filtered = hb_store:scope(Stores, [local, in_memory]),
    ?assertEqual(2, length(Filtered)).
 
scope_default_local_test() ->
    %% Stores without explicit scope default to 'local'
    Store = #{<<"store-module">> => hb_store_fs, <<"name">> => <<"no-scope">>},
    Filtered = hb_store:scope([Store], local),
    ?assertEqual(1, length(Filtered)).
 
filter_by_priority_test() ->
    Stores = [
        #{<<"store-module">> => hb_store_fs, <<"priority">> => 1},
        #{<<"store-module">> => hb_store_fs, <<"priority">> => 5},
        #{<<"store-module">> => hb_store_fs, <<"priority">> => 10}
    ],
    HighPriority = hb_store:filter(Stores, 
        fun(_Scope, Store) ->
            maps:get(<<"priority">>, Store, 0) >= 5
        end
    ),
    ?assertEqual(2, length(HighPriority)).
 
filter_by_module_test() ->
    Stores = [
        #{<<"store-module">> => hb_store_fs},
        #{<<"store-module">> => hb_store_lmdb},
        #{<<"store-module">> => hb_store_fs}
    ],
    FsOnly = hb_store:filter(Stores,
        fun(_Scope, Store) ->
            maps:get(<<"store-module">>, Store) =:= hb_store_fs
        end
    ),
    ?assertEqual(2, length(FsOnly)).
 
filter_single_store_test() ->
    %% filter accepts single store (converts to list)
    Store = #{<<"store-module">> => hb_store_fs, <<"keep">> => true},
    Result = hb_store:filter(Store, fun(_, S) -> maps:get(<<"keep">>, S, false) end),
    ?assertEqual(1, length(Result)).

7. sort/2

-spec sort(Stores, PreferenceOrder) -> SortedStores
    when
        Stores :: [map()],
        PreferenceOrder :: [atom()] | #{atom() => integer()},
        SortedStores :: [map()].

Description: Sort stores by scope preference. Accepts either a list of scopes (highest priority first) or a map of scope scores (higher score = higher priority). Unknown scopes default to score 0. Useful for prioritizing faster or cheaper stores.

Test Code:

-module(test_hb_store_sort).
-include_lib("eunit/include/eunit.hrl").
 
sort_by_preference_list_test() ->
    MemStore = #{<<"store-module">> => hb_store_fs, <<"scope">> => in_memory},
    LocalStore = #{<<"store-module">> => hb_store_fs, <<"scope">> => local},
    RemoteStore = #{<<"store-module">> => hb_store_gateway, <<"scope">> => remote},
    Stores = [RemoteStore, LocalStore, MemStore],
    
    %% Sort with in_memory first, then local, then remote
    Sorted = hb_store:sort(Stores, [in_memory, local, remote]),
    [First, Second, Third] = Sorted,
    ?assertEqual(in_memory, maps:get(<<"scope">>, First)),
    ?assertEqual(local, maps:get(<<"scope">>, Second)),
    ?assertEqual(remote, maps:get(<<"scope">>, Third)).
 
sort_by_score_map_test() ->
    LocalStore = #{<<"store-module">> => hb_store_fs, <<"scope">> => local},
    MemStore = #{<<"store-module">> => hb_store_fs, <<"scope">> => in_memory},
    Stores = [LocalStore, MemStore],
    
    %% Higher score = higher priority
    Scores = #{in_memory => 100, local => 50},
    Sorted = hb_store:sort(Stores, Scores),
    [First | _] = Sorted,
    ?assertEqual(in_memory, maps:get(<<"scope">>, First)).
 
sort_unknown_scope_defaults_zero_test() ->
    KnownStore = #{<<"store-module">> => hb_store_fs, <<"scope">> => local},
    UnknownStore = #{<<"store-module">> => hb_store_fs, <<"scope">> => custom_scope},
    Stores = [UnknownStore, KnownStore],
    
    %% Unknown scope gets score 0, local gets score 50
    Scores = #{local => 50},
    Sorted = hb_store:sort(Stores, Scores),
    [First | _] = Sorted,
    ?assertEqual(local, maps:get(<<"scope">>, First)).
 
sort_preserves_order_for_equal_scores_test() ->
    Store1 = #{<<"store-module">> => hb_store_fs, <<"scope">> => local, <<"id">> => 1},
    Store2 = #{<<"store-module">> => hb_store_fs, <<"scope">> => local, <<"id">> => 2},
    Stores = [Store1, Store2],
    
    %% Same scope means same score
    Sorted = hb_store:sort(Stores, #{local => 10}),
    ?assertEqual(2, length(Sorted)).

8. path/1, path/2, add_path/2, add_path/3, join/1

-spec path(Key) -> NormalizedPath
    when
        Key :: binary() | list(),
        NormalizedPath :: binary().
 
-spec path(Stores, Key) -> NormalizedPath
    when
        Stores :: map() | [map()],
        Key :: binary() | list(),
        NormalizedPath :: binary().
 
-spec add_path(Path1, Path2) -> JoinedPath
    when
        Path1 :: list(),
        Path2 :: list(),
        JoinedPath :: list().
 
-spec add_path(Stores, Path1, Path2) -> JoinedPath
    when
        Stores :: map() | [map()],
        Path1 :: list(),
        Path2 :: list(),
        JoinedPath :: list().
 
-spec join(PathParts) -> JoinedPath
    when
        PathParts :: [binary() | list()],
        JoinedPath :: binary().

Description: Path manipulation utilities. path/1,2 normalizes paths to binary format (uses hb_path:to_binary). add_path/2,3 concatenates path component lists. join/1 combines path parts with / separator. add_path/3 delegates to store's implementation if available, falls back to concatenation.

Test Code:

-module(test_hb_store_path).
-include_lib("eunit/include/eunit.hrl").
 
join_binary_parts_test() ->
    Parts = [<<"a">>, <<"b">>, <<"c">>],
    ?assertEqual(<<"a/b/c">>, hb_store:join(Parts)).
 
join_single_part_test() ->
    ?assertEqual(<<"single">>, hb_store:join([<<"single">>])).
 
join_empty_test() ->
    ?assertEqual(<<>>, hb_store:join([])).
 
path_from_list_test() ->
    Result = hb_store:path([<<"a">>, <<"b">>]),
    ?assertEqual(<<"a/b">>, Result).
 
path_from_binary_test() ->
    %% Binary input passes through
    Result = hb_store:path(<<"already/binary">>),
    ?assert(is_binary(Result)).
 
path_with_store_test() ->
    Store = hb_test_utils:test_store(hb_store_fs),
    Result = hb_store:path(Store, [<<"x">>, <<"y">>]),
    ?assertEqual(<<"x/y">>, Result).
 
add_path_concatenates_lists_test() ->
    Path1 = [<<"root">>, <<"dir">>],
    Path2 = [<<"file">>],
    Result = hb_store:add_path(Path1, Path2),
    ?assertEqual([<<"root">>, <<"dir">>, <<"file">>], Result).
 
add_path_empty_second_test() ->
    Path1 = [<<"base">>],
    Result = hb_store:add_path(Path1, []),
    ?assertEqual([<<"base">>], Result).
 
add_path_with_store_fallback_test() ->
    Store = hb_test_utils:test_store(hb_store_fs),
    Path1 = [<<"parent">>],
    Path2 = [<<"child">>],
    %% Store doesn't implement add_path, falls back to concatenation
    Result = hb_store:add_path(Store, Path1, Path2),
    ?assertEqual([<<"parent">>, <<"child">>], Result).

9. test_stores/0, generate_test_suite/1, generate_test_suite/2

-spec test_stores() -> [StoreOpts]
    when
        StoreOpts :: map().
 
-spec generate_test_suite(Suite) -> EUnitTests
    when
        Suite :: [{Description :: string(), TestFun :: fun((Store) -> any())}],
        EUnitTests :: term().
 
-spec generate_test_suite(Suite, Stores) -> EUnitTests
    when
        Suite :: [{Description :: string(), TestFun :: fun((Store) -> any())}],
        Stores :: [map()],
        EUnitTests :: term().

Description: Testing utilities for store implementations. test_stores/0 returns default test stores (fs, lmdb, lru, optionally rocksdb with benchmark-scale settings). generate_test_suite/1,2 creates EUnit {foreach, Setup, Teardown, Tests} fixtures that run same tests against multiple backends. Each test runs with start() setup and reset() teardown.

Test Code:

-module(test_hb_store_testing).
-include_lib("eunit/include/eunit.hrl").
 
test_stores_returns_list_test() ->
    Stores = hb_store:test_stores(),
    ?assert(is_list(Stores)),
    ?assert(length(Stores) >= 1),
    %% Each store has required store-module key
    lists:foreach(fun(Store) ->
        ?assert(maps:is_key(<<"store-module">>, Store))
    end, Stores).
 
test_stores_includes_fs_test() ->
    Stores = hb_store:test_stores(),
    FsStores = lists:filter(fun(S) ->
        maps:get(<<"store-module">>, S) =:= hb_store_fs
    end, Stores),
    ?assert(length(FsStores) >= 1).
 
test_stores_has_benchmark_scale_test() ->
    Stores = hb_store:test_stores(),
    %% At least one store should have benchmark-scale
    HasScale = lists:any(fun(S) ->
        maps:is_key(<<"benchmark-scale">>, S)
    end, Stores),
    ?assert(HasScale).
 
generate_test_suite_returns_list_test() ->
    Suite = [{"dummy test", fun(_Store) -> ok end}],
    Result = hb_store:generate_test_suite(Suite),
    ?assert(is_list(Result)).
 
%% Example: test suite that runs against all default backends
store_operations_suite_test_() ->
    hb_store:generate_test_suite([
        {"write and read", fun(Store) ->
            ?assertEqual(ok, hb_store:write(Store, <<"k">>, <<"v">>)),
            ?assertEqual({ok, <<"v">>}, hb_store:read(Store, <<"k">>))
        end},
        {"read not found", fun(Store) ->
            ?assertEqual(not_found, hb_store:read(Store, <<"missing">>))
        end},
        {"make group and list", fun(Store) ->
            ?assertEqual(ok, hb_store:make_group(Store, <<"g">>)),
            ?assertEqual(ok, hb_store:write(Store, [<<"g">>, <<"x">>], <<"val">>)),
            {ok, Items} = hb_store:list(Store, <<"g">>),
            ?assert(lists:member(<<"x">>, Items))
        end}
    ]).
 
%% Example: custom stores list
generate_test_suite_custom_stores_test_() ->
    CustomStores = [hb_test_utils:test_store(hb_store_fs)],
    hb_store:generate_test_suite(
        [{"custom store write", fun(Store) ->
            ?assertEqual(ok, hb_store:write(Store, <<"c">>, <<"d">>))
        end}],
        CustomStores
    ).

Store Behavior

Required Callbacks

-callback start(StoreOpts) -> ok | {ok, Instance} | {error, Reason}.
-callback stop(StoreOpts) -> ok.
-callback reset(StoreOpts) -> ok.
-callback read(StoreOpts, Key) -> {ok, Value} | not_found.
-callback write(StoreOpts, Key, Value) -> ok.
-callback type(StoreOpts, Key) -> simple | composite | not_found.
-callback list(StoreOpts, Key) -> {ok, [Key]} | not_found.
-callback make_group(StoreOpts, Key) -> ok.
-callback make_link(StoreOpts, Existing, New) -> ok.
-callback path(StoreOpts, Key) -> NormalizedPath.
-callback add_path(StoreOpts, Base, Part) -> JoinedPath.

Optional Callbacks

-callback scope() -> Scope.
-callback scope(StoreOpts) -> Scope.
-callback match(StoreOpts, Pattern) -> {ok, [Key]} | not_found.
-callback resolve(StoreOpts, Key) -> ResolvedKey.

Access Control

Access Policies

% Policy definitions
#{
    <<"read">> => [read, resolve, list, type, path, add_path, join],
    <<"write">> => [write, make_link, make_group, reset, path, add_path, join],
    <<"admin">> => [start, stop, reset]
}

Configuration

% Read-only store
#{
    <<"store-module">> => hb_store_fs,
    <<"name">> => <<"readonly">>,
    <<"access">> => [<<"read">>]
}
 
% Write-only store
#{
    <<"store-module">> => hb_store_fs,
    <<"name">> => <<"writeonly">>,
    <<"access">> => [<<"write">>]
}
 
% Full access (default)
#{
    <<"store-module">> => hb_store_fs,
    <<"name">> => <<"fullaccess">>,
    <<"access">> => [<<"read">>, <<"write">>, <<"admin">>]
}
 
% No access restrictions (no access key)
#{
    <<"store-module">> => hb_store_fs,
    <<"name">> => <<"unrestricted">>
}

Test Code:

-module(test_hb_store_access).
-include_lib("eunit/include/eunit.hrl").
 
read_only_store_test() ->
    ReadOnly = #{
        <<"store-module">> => hb_store_fs,
        <<"name">> => <<"ro">>,
        <<"access">> => [<<"read">>]
    },
    WriteStore = hb_test_utils:test_store(hb_store_fs),
    Chain = [ReadOnly, WriteStore],
    
    hb_store:start(Chain),
    % Write bypasses ReadOnly, goes to WriteStore
    ?assertEqual(ok, hb_store:write(Chain, <<"key">>, <<"val">>)),
    % Read works through chain
    ?assertMatch({ok, <<"val">>}, hb_store:read(Chain, <<"key">>)),
    % ReadOnly doesn't have the value
    ?assertEqual(not_found, hb_store:read([ReadOnly], <<"key">>)).

Store Chains

Fallback Pattern

% Try stores in order until one succeeds
Stores = [
    FastStore,      % Try fastest first
    LocalStore,     % Then local
    RemoteStore     % Finally remote
],
 
{ok, Value} = hb_store:read(Stores, Key).
% Reads from first store that has the key

Tiered Storage

% Memory -> Local -> Remote
Stores = [
    #{<<"store-module">> => hb_store_memory, <<"scope">> => in_memory},
    #{<<"store-module">> => hb_store_fs, <<"scope">> => local},
    #{<<"store-module">> => hb_store_gateway, <<"scope">> => remote}
],
 
% Read checks in order
{ok, Value} = hb_store:read(Stores, Key).
 
% Write goes to first writable store
ok = hb_store:write(Stores, Key, Value).

Specialized Chains

% Cache + Archive
ReadChain = [CacheStore, ArchiveStore],
WriteChain = [CacheStore],  % Only cache writes
 
{ok, Value} = hb_store:read(ReadChain, Key),
ok = hb_store:write(WriteChain, Key, NewValue).

Store Scopes

Scope Types

Scope	Description	Use Case
`in_memory`	RAM-based storage	Cache, temporary data
`local`	Local disk storage	Persistent local data
`remote`	Network storage	Distributed data
`arweave`	Arweave network	Permanent storage

Scope Filtering

% Get only local stores
LocalStores = hb_store:scope(Opts, local),
 
% Get local and remote
Stores = hb_store:scope(Opts, [local, remote]),
 
% Custom filter
FastStores = hb_store:filter(Stores,
    fun(Scope, _Store) ->
        lists:member(Scope, [in_memory, local])
    end
).

Common Patterns

%% Simple read/write
Store = #{<<"store-module">> => hb_store_fs, <<"name">> => <<"data">>},
hb_store:start(Store),
ok = hb_store:write(Store, <<"key">>, <<"value">>),
{ok, <<"value">>} = hb_store:read(Store, <<"key">>).
 
%% Nested keys
ok = hb_store:make_group(Store, <<"group">>),
ok = hb_store:write(Store, [<<"group">>, <<"item">>], <<"data">>),
{ok, [<<"item">>]} = hb_store:list(Store, <<"group">>).
 
%% Symbolic links
ok = hb_store:write(Store, <<"target">>, <<"value">>),
ok = hb_store:make_link(Store, <<"target">>, <<"alias">>),
{ok, <<"value">>} = hb_store:read(Store, <<"alias">>).
 
%% Store chains with fallback
Stores = [FastCache, SlowDisk, RemoteBackup],
case hb_store:read(Stores, Key) of
    {ok, Value} -> Value;
    not_found -> default_value()
end.
 
%% Scope-filtered stores
Opts = #{store => [MemStore, FSStore, GatewayStore]},
LocalOnly = hb_store:scope(Opts, [in_memory, local]),
{ok, Value} = hb_store:read(LocalOnly, Key).
 
%% Access-controlled stores
PublicRead = #{
    <<"store-module">> => hb_store_fs,
    <<"name">> => <<"public">>,
    <<"access">> => [<<"read">>]
},
PrivateWrite = #{
    <<"store-module">> => hb_store_fs,
    <<"name">> => <<"private">>,
    <<"access">> => [<<"write">>]
},
Chain = [PublicRead, PrivateWrite],
% Reads from public, writes to private
ok = hb_store:write(Chain, Key, Value).

Performance Benchmarking

Benchmark Functions

% Run comprehensive benchmarks
hb_store:generate_test_suite(StoreModule).
 
% Custom benchmarks
hb_store:generate_test_suite(StoreModule, CustomOpts).

Metrics Measured

Write operations per second
Read operations per second
List operations performance
Large message handling
Storage overhead

References

Filesystem Store - hb_store_fs.erl
Gateway Store - hb_store_gateway.erl
LMDB Store - hb_store_lmdb.erl
Cache System - hb_cache.erl

Notes

Pluggable: Easy to add new storage backends
Chain Support: Multiple stores with automatic fallback
Access Control: Fine-grained permission policies
Scope Filtering: Filter by storage location/type
Path Abstraction: Unified path handling across stores
Link Support: Symbolic links for key aliasing
Instance Caching: Store instances cached in persistent_term
Composite Keys: Support for hierarchical/grouped keys
Not Found Handling: Consistent not_found return across stores
Behavior-Based: Clean callback interface for implementations