hb_http.erl - HyperBEAM HTTP Request/Reply Core

Overview

Purpose: Core HTTP request/reply functionality for HyperBEAM
Module: hb_http
Protocol: HTTP/HTTPS with message-based communication
Pattern: Message request → HTTP → Message response

This module provides HyperBEAM's primary HTTP functionality, converting messages to HTTP requests and processing responses back into message form. It handles both client requests (GET, POST) and server-side request processing, supporting multiple codec devices and serialization formats.

Core Responsibilities

Client Requests: GET/POST operations to remote nodes
Message Conversion: Transform HyperBEAM messages to/from HTTP
Codec Support: Handle httpsig@1.0, ans104@1.0, and structured messages
Cookie Management: Parse and normalize Set-Cookie headers
Response Processing: Convert HTTP responses to appropriate message formats
Multi-Request: Delegate to hb_http_multi for route-based requests

Dependencies

HyperBEAM: hb_ao, hb_message, hb_util, hb_maps, hb_opts, hb_json, hb_singleton
Codecs: dev_codec_ans104, dev_codec_structured, dev_codec_cookie, dev_codec_httpsig
HTTP Client: hb_http_client, hb_http_multi
Arweave: ar_bundles, ar_wallet
External: cowboy_req, httpc
Includes: include/hb.hrl

Public Functions Overview

%% Initialization
-spec start() -> ok.
 
%% Client Requests
-spec get(Node, Opts) -> {ok, Message} | {error, Reason}.
-spec get(Node, Path, Opts) -> {ok, Message} | {error, Reason}.
-spec get(Node, Message, Opts) -> {ok, Message} | {error, Reason}.
 
-spec post(Node, Path, Opts) -> {ok, Message} | {error, Reason}.
-spec post(Node, Message, Opts) -> {ok, Message} | {error, Reason}.
-spec post(Node, Path, Message, Opts) -> {ok, Message} | {error, Reason}.
 
%% Generic Request
-spec request(Message, Opts) -> {ok | created | error | failure, Result}.
-spec request(Method, Peer, Path, Opts) -> {ok | created | error | failure, Result}.
-spec request(Method, Peer, Path, Message, Opts) -> {ok | created | error | failure, Result}.
 
%% Message Conversion
-spec message_to_request(Message, Opts) -> {ok, Method, Peer, Path, Message, Opts}.
 
%% Server Processing
-spec reply(Request, StatusCode, Headers, Body) -> cowboy_req:req().
-spec accept_to_codec(Accept, Opts) -> CodecDevice.
 
%% Utilities
-spec req_to_tabm_singleton(Request, Message, Opts) -> Messages.

Public Functions

1. start/0

-spec start() -> ok.

Description: Initialize HTTP client by disabling keep-alive connections.

Test Code:

-module(hb_http_test).
-include_lib("eunit/include/eunit.hrl").
 
%% Helper to start a mock HTTP server
start_mock_server(ResponseBody, StatusCode) ->
    {ok, ListenSock} = gen_tcp:listen(0, [binary, {active, false}, {reuseaddr, true}]),
    {ok, Port} = inet:port(ListenSock),
    spawn(fun() -> mock_server_loop(ListenSock, ResponseBody, StatusCode) end),
    {Port, ListenSock}.
 
mock_server_loop(ListenSock, ResponseBody, StatusCode) ->
    case gen_tcp:accept(ListenSock, 5000) of
        {ok, Sock} ->
            {ok, _Request} = gen_tcp:recv(Sock, 0, 5000),
            ContentLength = integer_to_binary(byte_size(ResponseBody)),
            Response = <<"HTTP/1.1 ", (integer_to_binary(StatusCode))/binary, " OK\r\n",
                        "Content-Type: application/json\r\n",
                        "Content-Length: ", ContentLength/binary, "\r\n\r\n",
                        ResponseBody/binary>>,
            gen_tcp:send(Sock, Response),
            gen_tcp:close(Sock),
            mock_server_loop(ListenSock, ResponseBody, StatusCode);
        {error, timeout} ->
            gen_tcp:close(ListenSock);
        {error, _} ->
            gen_tcp:close(ListenSock)
    end.
 
start_test() ->
    ?assertEqual(ok, hb_http:start()).

2. get/2, get/3

-spec get(Node, Opts) -> {ok, Message} | {error, Reason}
    when
        Node :: binary(),
        Opts :: map(),
        Message :: map(),
        Reason :: term().
 
-spec get(Node, PathOrMessage, Opts) -> {ok, Message} | {error, Reason}
    when
        Node :: binary(),
        PathOrMessage :: binary() | map(),
        Opts :: map(),
        Message :: map(),
        Reason :: term().

Description: Perform HTTP GET request to a remote node, returning deserialized message response.

Test Code:

get_basic_test() ->
    {Port, Sock} = start_mock_server(<<"{\"key\":\"value\"}">>, 200),
    URL = iolist_to_binary([<<"http://127.0.0.1:">>, integer_to_binary(Port)]),
    
    {ok, Res} = hb_http:get(URL, <<"/">>, #{http_client => httpc}),
    ?assert(is_map(Res)),
    ?assertEqual(200, maps:get(<<"status">>, Res)),
    
    gen_tcp:close(Sock).
 
get_with_path_test() ->
    {Port, Sock} = start_mock_server(<<"{\"data\":123}">>, 200),
    URL = iolist_to_binary([<<"http://127.0.0.1:">>, integer_to_binary(Port)]),
    
    {ok, Res} = hb_http:get(URL, <<"/api/data">>, #{http_client => httpc}),
    ?assert(is_map(Res)),
    
    gen_tcp:close(Sock).

3. post/3, post/4

-spec post(Node, PathOrMessage, Opts) -> {ok, Message} | {error, Reason}
    when
        Node :: binary(),
        PathOrMessage :: binary() | map(),
        Opts :: map(),
        Message :: map(),
        Reason :: term().
 
-spec post(Node, Path, Message, Opts) -> {ok, Message} | {error, Reason}
    when
        Node :: binary(),
        Path :: binary(),
        Message :: map(),
        Opts :: map(),
        Reason :: term().

Description: Post message to remote node via HTTP, returning deserialized response.

Test Code:

post_basic_test() ->
    {Port, Sock} = start_mock_server(<<"{\"result\":\"ok\"}">>, 200),
    URL = iolist_to_binary([<<"http://127.0.0.1:">>, integer_to_binary(Port)]),
    
    {ok, Res} = hb_http:post(URL, <<"/">>, #{http_client => httpc}),
    ?assert(is_map(Res)),
    ?assertEqual(200, maps:get(<<"status">>, Res)),
    
    gen_tcp:close(Sock).
 
post_with_message_test() ->
    {Port, Sock} = start_mock_server(<<"{\"saved\":true}">>, 201),
    URL = iolist_to_binary([<<"http://127.0.0.1:">>, integer_to_binary(Port)]),
    
    Message = #{<<"key">> => <<"value">>},
    {ok, Res} = hb_http:post(URL, <<"/api/save">>, Message, #{http_client => httpc}),
    ?assert(is_map(Res)),
    
    gen_tcp:close(Sock).

4. request/2, request/4, request/5

-spec request(Message, Opts) -> {Status, Result}
    when
        Message :: map(),
        Opts :: map(),
        Status :: ok | created | error | failure,
        Result :: term().
 
-spec request(Method, Peer, Path, Message, Opts) -> {Status, Result}
    when
        Method :: binary(),
        Peer :: binary() | map(),
        Path :: binary(),
        Message :: map(),
        Opts :: map(),
        Status :: ok | created | error | failure,
        Result :: term().

Description: Generic HTTP request handler supporting GET, POST, PUT, DELETE. Handles routing, multi-requests, codec selection, and response processing.

Special Cases:

Routes: Delegates to hb_http_multi when <<"nodes">> present
URI Options: Merges additional options from route configuration
AO-Result: Returns single key value when ao-result header present

Test Code:

request_get_test() ->
    {Port, Sock} = start_mock_server(<<"{\"method\":\"GET\"}">>, 200),
    URL = iolist_to_binary([<<"http://127.0.0.1:">>, integer_to_binary(Port)]),
    
    {ok, Res} = hb_http:request(<<"GET">>, URL, <<"/">>, #{}, #{http_client => httpc}),
    ?assert(is_map(Res)),
    ?assertEqual(200, maps:get(<<"status">>, Res)),
    
    gen_tcp:close(Sock).
 
request_post_test() ->
    {Port, Sock} = start_mock_server(<<"{\"method\":\"POST\"}">>, 201),
    URL = iolist_to_binary([<<"http://127.0.0.1:">>, integer_to_binary(Port)]),
    
    {ok, Res} = hb_http:request(<<"POST">>, URL, <<"/data">>, #{<<"key">> => <<"val">>}, #{http_client => httpc}),
    ?assert(is_map(Res)),
    
    gen_tcp:close(Sock).
 
request_error_test() ->
    {Port, Sock} = start_mock_server(<<"{\"error\":\"not found\"}">>, 404),
    URL = iolist_to_binary([<<"http://127.0.0.1:">>, integer_to_binary(Port)]),
    
    {error, Res} = hb_http:request(<<"GET">>, URL, <<"/missing">>, #{}, #{http_client => httpc}),
    ?assert(is_map(Res)),
    ?assertEqual(404, maps:get(<<"status">>, Res)),
    
    gen_tcp:close(Sock).
 
request_failure_test() ->
    {Port, Sock} = start_mock_server(<<"{\"error\":\"internal\"}">>, 500),
    URL = iolist_to_binary([<<"http://127.0.0.1:">>, integer_to_binary(Port)]),
    
    {failure, Res} = hb_http:request(<<"GET">>, URL, <<"/fail">>, #{}, #{http_client => httpc}),
    ?assert(is_map(Res)),
    ?assertEqual(500, maps:get(<<"status">>, Res)),
    
    gen_tcp:close(Sock).

5. message_to_request/2

-spec message_to_request(Message, Opts) -> {ok, Method, Peer, Path, Message, Opts}
    when
        Message :: map(),
        Opts :: map(),
        Method :: binary(),
        Peer :: binary(),
        Path :: binary().

Description: Convert HyperBEAM message to HTTP request components, extracting method, peer, path, and preparing message for transmission.

Test Code:

message_to_request_basic_test() ->
    Message = #{
        <<"url">> => <<"http://example.com/api/data">>,
        <<"method">> => <<"POST">>,
        <<"body">> => <<"test">>
    },
    {ok, Method, Peer, Path, _Msg, _Opts} = hb_http:message_to_request(Message, #{}),
    ?assertEqual(<<"POST">>, Method),
    ?assertEqual(<<"http://example.com">>, Peer),
    ?assertEqual(<<"/api/data">>, Path).
 
message_to_request_default_method_test() ->
    Message = #{
        <<"url">> => <<"http://example.com/path">>
    },
    {ok, Method, _Peer, _Path, _Msg, _Opts} = hb_http:message_to_request(Message, #{}),
    ?assertEqual(<<"GET">>, Method).

6. reply/4

-spec reply(Request, StatusCode, Headers, Body) -> cowboy_req:req()
    when
        Request :: cowboy_req:req(),
        StatusCode :: integer(),
        Headers :: map() | list(),
        Body :: binary().

Description: Send HTTP response to client via Cowboy, handling headers, status codes, and body serialization.

Note: This function requires a Cowboy request object and is tested through integration tests with hb_http_server.

Features:

Auto-detects content type from body
Converts headers map to list format
Handles binary and JSON bodies
Sets appropriate CORS headers

7. accept_to_codec/2

-spec accept_to_codec(Accept, Opts) -> CodecDevice
    when
        Accept :: binary() | undefined,
        CodecDevice :: binary().

Description: Convert HTTP Accept header to HyperBEAM codec device identifier.

Test Code:

accept_to_codec_json_test() ->
    ?assertEqual(<<"structured@1.0">>, hb_http:accept_to_codec(<<"application/json">>, #{})).
 
accept_to_codec_octet_stream_test() ->
    ?assertEqual(<<"ans104@1.0">>, hb_http:accept_to_codec(<<"application/octet-stream">>, #{})).
 
accept_to_codec_text_plain_test() ->
    ?assertEqual(<<"structured@1.0">>, hb_http:accept_to_codec(<<"text/plain">>, #{})).
 
accept_to_codec_undefined_test() ->
    ?assertEqual(<<"structured@1.0">>, hb_http:accept_to_codec(undefined, #{})).

8. req_to_tabm_singleton/3

-spec req_to_tabm_singleton(Request, Message, Opts) -> Messages
    when
        Request :: cowboy_req:req(),
        Message :: map(),
        Opts :: map(),
        Messages :: [map()].

Description: Convert Cowboy request to HyperBEAM message sequence for singleton execution.

Note: This function requires a Cowboy request object and is tested through integration tests with hb_http_server.

Processing Steps:

Extract HTTP method and path
Parse query parameters
Extract and normalize headers
Handle cookie parsing
Process request body based on content-type
Detect codec device from Accept header
Add peer information if ao-peer-port present

Response Status Mapping

response_status_to_atom(Status) ->
    case Status of
        201 -> created;
        X when X < 400 -> ok;
        X when X < 500 -> error;
        _ -> failure
    end.

Status Categories:

created - HTTP 201
ok - HTTP 2xx-3xx (except 201)
error - HTTP 4xx (client errors)
failure - HTTP 5xx (server errors)

Codec Device Handling

Outbound Requests

% Detect codec from message or use default
CodecDev = hb_maps:get(<<"codec-device">>, Message, <<"httpsig@1.0">>, Opts),
 
% Prepare request based on codec
prepare_request(CodecDev, Method, Peer, Path, Message, Opts)

Inbound Responses

% Find codec from response headers
CodecDev = hb_maps:get(<<"codec-device">>, Headers, <<"httpsig@1.0">>, Opts),
 
% Convert response to message
outbound_result_to_message(CodecDev, Status, Headers, Body, Opts)

Supported Codecs:

httpsig@1.0 - HTTP Signature (default)
ans104@1.0 - ANS-104 data items
structured@1.0 - Structured messages
json@1.0 - JSON encoding

Cookie Handling

Set-Cookie Processing

% Extract Set-Cookie headers from response
SetCookieLines = [{<<"set-cookie">>, KeyVal} || {<<"set-cookie">>, KeyVal} <- Headers],
 
% Convert to cookie message
{ok, MsgWithCookies} = dev_codec_cookie:from(
    #{<<"set-cookie">> => SetCookieLines},
    #{},
    Opts
),
 
% Merge into response headers
HeaderMap = hb_maps:merge(hb_maps:from_list(Headers), MsgWithCookies, Opts)

Cookie Request Handling

% Parse cookie from request
{ok, WithCookie} = case maps:get(<<"cookie">>, RawHeaders, undefined) of
    undefined -> {ok, BaseMsg};
    Cookie -> dev_codec_cookie:from(BaseMsg#{<<"cookie">> => Cookie}, Req, Opts)
end

Common Patterns

%% Simple GET request
{ok, Response} = hb_http:get(<<"http://node.example.com">>, <<"/path">>, #{}).
 
%% POST with message
Message = #{<<"key">> => <<"value">>},
{ok, Response} = hb_http:post(Node, <<"/endpoint">>, Message, #{}).
 
%% Signed POST request
Wallet = hb:wallet(),
SignedMsg = hb_message:commit(Message, #{priv_wallet => Wallet}),
{ok, Response} = hb_http:post(Node, SignedMsg, #{}).
 
%% Request with specific codec
Opts = #{<<"codec-device">> => <<"ans104@1.0">>},
{ok, Response} = hb_http:post(Node, Message, Opts).
 
%% Extract single result key
Opts = #{http_only_result => true},
Message = #{<<"path">> => <<"/data">>, <<"ao-result">> => <<"body">>},
{ok, BodyOnly} = hb_http:post(Node, Message, Opts).
 
%% Handle routes (multi-request)
Config = #{
    <<"nodes">> => [
        <<"http://node1.example.com">>,
        <<"http://node2.example.com">>
    ]
},
{ok, Response} = hb_http:request(<<"GET">>, Config, <<"/path">>, #{}, #{}).

Configuration Options

Opts = #{
    % HTTP client selection
    http_client => gun | httpc,  % Default: gun
    
    % Result extraction
    http_only_result => true | false,  % Default: true
    
    % Codec selection
    <<"codec-device">> => <<"httpsig@1.0">> | <<"ans104@1.0">> | <<"structured@1.0">>,
    
    % Wallet for signing
    priv_wallet => {PrivateKey, PublicKey},
    
    % Request timeout
    http_request_send_timeout => Milliseconds,
    
    % Response size limit
    limit => Bytes | infinity,
    
    % Peer identification
    is_peer_request => boolean(),
    
    % HTTP monitoring
    http_monitor => MonitorMessage,
    http_reference => ReferenceValue,
    
    % Cache control
    cache_control => [<<"no-cache">>, <<"no-store">>]
}.

Error Handling

% Client errors (4xx)
{error, Reason}
 
% Server errors (5xx)
{failure, Reason}
 
% Network errors
{error, client_error}
{error, connection_closed}
{error, timeout}
 
% Codec errors
{error, unsupported_codec}
{error, invalid_message}

Server-Side Processing

Request Normalization

% Convert Cowboy request to HyperBEAM message
Messages = hb_http:req_to_tabm_singleton(Req, InitialMsg, Opts),
 
% Process through singleton
Results = hb_ao:resolve_many(Messages, Opts)

Response Generation

% Send response back to client
Reply = hb_http:reply(Req, 200, Headers, Body)

Performance Notes

Keep-Alive Disabled: Connections don't persist between requests
Parallel Requests: Use hb_http_multi for concurrent node access
Streaming: Supports chunked transfer encoding via gun
Metrics: Records request duration in Prometheus when enabled
Connection Pooling: Managed by hb_http_client gen_server

References

HTTP Client - hb_http_client.erl
Multi-Request - hb_http_multi.erl
Message System - hb_message.erl, hb_ao.erl
Codecs - dev_codec_*.erl modules
Server - hb_http_server.erl

Notes

Default Codec: httpsig@1.0 used unless specified
Message Signing: Optional, controlled by wallet presence
Cookie Support: Full Set-Cookie and Cookie header processing
CORS: Automatic CORS headers on responses
Query Parameters: Extracted and added to message
Path Resolution: Supports nested path traversal in messages
AO-Result: Can extract single key from response for efficiency
Peer Detection: Honors x-real-ip for accurate peer address
Body Streaming: Large responses handled efficiently
Status Normalization: Consistent status atom mapping
Multi-Gateway: Seamless delegation to multi-request system
Header Filtering: Default filter for content-length
Content-Type Detection: Auto-detects from body or headers
Accept Header: Maps to appropriate codec device
Monitoring: Optional HTTP request monitoring via AO-Core# hb_http.erl - HyperBEAM HTTP Request/Reply Core