diff --git a/CHANGELOG.md b/CHANGELOG.md index 4242ebd..2217d34 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +# 0.9.0 + +* Added the `unix_socket` request option to both clients and connection managers. + # 0.8.0 ## Breaking Changes diff --git a/README.md b/README.md index a9f4556..089ce53 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ An HTTP Client library built around [Mint](https://github.com/elixir-mint/mint), ```elixir {:ok, resp} = Pepper.HTTP.Client.request(method, url, headers, body, options) -{:ok, resp} = Pepper.HTTP.Client.request(:get, "https://example.com", [{"user-agent", "pepper-http/0.6.0"}], nil, []) +{:ok, resp} = Pepper.HTTP.Client.request(:get, "https://example.com", [{"user-agent", "pepper-http/0.9.0"}], nil, []) resp.status_code # => 200 resp.headers # => [{"content-type", "text/plain"}, {"content-length", "12"}] diff --git a/docs/clients.md b/docs/clients.md index 5fab6f2..e2fad5e 100644 --- a/docs/clients.md +++ b/docs/clients.md @@ -45,6 +45,33 @@ case result do end ``` +### Unix Domain Sockets + +Both clients accept `unix_socket: "/path/to/service.sock"` with either connection manager: + +```elixir +Pepper.HTTP.ContentClient.get( + "http://localhost/api/status", + [], + [], + unix_socket: "/tmp/service.sock" +) +``` + +Pepper connects to the socket file instead of resolving the URL's host or opening a TCP +connection. The URL still supplies the HTTP scheme, host, port, request path, and query. +For example, the request above sends `GET /api/status` with `Host: localhost`. +HTTP/1 requests preserve an explicitly supplied `Host` header. + +`http://` sends plain HTTP over the socket; `https://` uses TLS over the socket, with the +URL's hostname used for certificate verification and SNI unless `connect_options[:hostname]` +overrides it. Existing `connect_options` and timeouts apply. Proxy options cannot be +combined with `unix_socket`. + +The socket path must be a non-empty string. Omit `unix_socket`, or set it to `nil`, to use +the usual TCP connection. Unix sockets require support from the operating system. +Pooled connections are kept separate by socket path, URL destination, and connection options. + ### ContentClient Usage If you are in need of something more featured, Pepper provides a ContentClient which can handle most common request body types and response bodies. diff --git a/lib/pepper/http/client.ex b/lib/pepper/http/client.ex index b84797d..616ba3b 100644 --- a/lib/pepper/http/client.ex +++ b/lib/pepper/http/client.ex @@ -5,6 +5,10 @@ defmodule Pepper.HTTP.Client do This is a simple base client and doesn't handle any special content-types on request or response. Use the Pepper.HTTP.ContentClient instead if you need content-type handling. + + Pass `unix_socket: "/path/to/service.sock"` to connect through a Unix domain socket. + The URL still specifies the HTTP host, port, path, and query. Both one-off and + pooled connections support this option. """ require Logger @@ -40,6 +44,7 @@ defmodule Pepper.HTTP.Client do | {:response_body_handler_options, any()} | {:attempts, non_neg_integer()} | {:connect_options, [connect_option()]} + | {:unix_socket, String.t() | nil} @type uri_or_url :: URI.t() | String.t() @@ -200,9 +205,18 @@ defmodule Pepper.HTTP.Client do :recv_size, :recv_timeout, :connect_timeout, - :connect_options + :connect_options, + :unix_socket ] + defp validate_options!([{:unix_socket, path} = pair | rest], acc) do + unless is_nil(path) or (is_binary(path) and path != "" and not String.contains?(path, <<0>>)) do + raise ArgumentError, ":unix_socket must be a non-empty path string or nil" + end + + validate_options!(rest, [pair | acc]) + end + defp validate_options!( [{key, _} = pair | rest], acc ) when key in @allowed_keys do diff --git a/lib/pepper/http/connection_manager/one_off.ex b/lib/pepper/http/connection_manager/one_off.ex index 83ec790..2e6459d 100644 --- a/lib/pepper/http/connection_manager/one_off.ex +++ b/lib/pepper/http/connection_manager/one_off.ex @@ -18,18 +18,7 @@ defmodule Pepper.HTTP.ConnectionManager.OneOff do @spec request(term(), Request.t()) :: {:ok, Response.t()} | {:error, error_reasons()} def request(_id, %Request{} = request) do - connect_options = - Keyword.merge( - [ - mode: :passive, - transport_opts: [ - timeout: request.options[:connect_timeout], - ], - ], - Keyword.get(request.options, :connect_options, []) - ) - - case Mint.HTTP.connect(request.scheme, request.uri.host, request.uri.port, connect_options) do + case connect(request, :passive) do {:ok, conn} -> {request, is_stream?, body} = determine_if_body_should_stream(conn, request) diff --git a/lib/pepper/http/connection_manager/pooled.ex b/lib/pepper/http/connection_manager/pooled.ex index 4e470fc..1013c78 100644 --- a/lib/pepper/http/connection_manager/pooled.ex +++ b/lib/pepper/http/connection_manager/pooled.ex @@ -51,8 +51,9 @@ defmodule Pepper.HTTP.ConnectionManager.Pooled do alias Pepper.HTTP.ConnectError import Pepper.HTTP.Utils, only: [safe_reduce_ets_table: 3] + import Pepper.HTTP.ConnectionManager.Utils, only: [connection_key: 1] - @type conn_key :: {scheme::atom(), host::String.t(), port::integer(), Keyword.t()} + @type conn_key :: Pepper.HTTP.ConnectionManager.Utils.conn_key() @type connection_id :: GenServer.server() @@ -213,9 +214,7 @@ defmodule Pepper.HTTP.ConnectionManager.Pooled do def handle_call({:request, %Pepper.HTTP.Request{} = request}, from, %State{} = state) do start_time = System.monotonic_time(:microsecond) request = %{request | time: start_time} - connect_options = Keyword.get(request.options, :connect_options, []) - - key = {request.scheme, request.uri.host, request.uri.port, connect_options} + key = connection_key(request) case checkout_connection(key, state) do {:empty, state} -> ex = %CheckoutError{ diff --git a/lib/pepper/http/connection_manager/pooled_connection.ex b/lib/pepper/http/connection_manager/pooled_connection.ex index ec01dd9..b3f8187 100644 --- a/lib/pepper/http/connection_manager/pooled_connection.ex +++ b/lib/pepper/http/connection_manager/pooled_connection.ex @@ -6,9 +6,7 @@ defmodule Pepper.HTTP.ConnectionManager.PooledConnection do lifespan: nil, pool_pid: nil, conn: nil, - scheme: nil, - host: nil, - port: nil, + connection_key: nil, status: :ok, just_reconnected: false, # @@ -284,10 +282,8 @@ defmodule Pepper.HTTP.ConnectionManager.PooledConnection do state Mint.HTTP.open?(state.conn) -> - # ensure that connection's scheme, host and port matches the request - if request.scheme == state.scheme and - request.uri.host == state.host and - request.uri.port == state.port do + # Reclaimed workers must reconnect when the destination or options change. + if connection_key(request) == state.connection_key do # it matches, so return the state as is state else @@ -302,25 +298,12 @@ defmodule Pepper.HTTP.ConnectionManager.PooledConnection do if state.conn do {:ok, state} else - req_options = request.options - connect_options = Keyword.merge( - [ - mode: :active, - transport_opts: [ - timeout: req_options[:connect_timeout], - ], - ], - Keyword.get(request.options, :connect_options, []) - ) - - case Mint.HTTP.connect(request.scheme, request.uri.host, request.uri.port, connect_options) do + case connect(request, :active) do {:ok, conn} -> {:ok, %{ state | conn: conn, - scheme: request.scheme, - host: request.uri.host, - port: request.uri.port, + connection_key: connection_key(request), just_reconnected: true } } diff --git a/lib/pepper/http/connection_manager/utils.ex b/lib/pepper/http/connection_manager/utils.ex index 439e718..95057f7 100644 --- a/lib/pepper/http/connection_manager/utils.ex +++ b/lib/pepper/http/connection_manager/utils.ex @@ -8,6 +8,60 @@ defmodule Pepper.HTTP.ConnectionManager.Utils do @type conn :: Mint.Core.Conn.conn() + @type conn_key :: {atom(), String.t(), integer(), String.t() | nil, Keyword.t()} + + @spec connection_key(Request.t()) :: conn_key() + def connection_key(%Request{} = request) do + {request.scheme, request.uri.host, request.uri.port, Keyword.get(request.options, :unix_socket), + Keyword.get(request.options, :connect_options, [])} + end + + @spec connect(Request.t(), mode()) :: {:ok, conn()} | {:error, term()} + def connect(%Request{} = request, mode) do + options = + Keyword.merge( + [mode: mode, transport_opts: [timeout: request.options[:connect_timeout]]], + Keyword.get(request.options, :connect_options, []) + ) + + case Keyword.get(request.options, :unix_socket) do + nil -> + Mint.HTTP.connect(request.scheme, request.uri.host, request.uri.port, options) + + path -> + if Keyword.has_key?(options, :proxy) do + raise ArgumentError, ":unix_socket cannot be combined with a proxy" + end + + options = Keyword.put_new(options, :hostname, request.uri.host) + + case Mint.HTTP.connect(request.scheme, {:local, path}, 0, options) do + {:ok, conn} -> + {:ok, restore_http_port(conn, request.uri.port)} + + {:error, _reason} = error -> + error + end + end + end + + # Unix sockets require transport port 0. Mint also uses that port for HTTP + # identity, so restore the logical URL port before sending any requests. + defp restore_http_port(%Mint.HTTP1{} = conn, port) do + %{conn | port: port} + end + + defp restore_http_port(%Mint.HTTP2{} = conn, port) do + authority = + if URI.default_port(conn.scheme) == port do + conn.hostname + else + "#{conn.hostname}:#{port}" + end + + %{conn | port: port, authority: authority} + end + @spec timespan(function(), System.time_unit()) :: {{start_at::integer(), end_at::integer()}, result::any()} def timespan(callback, unit \\ :microsecond) when is_function(callback, 0) do diff --git a/mix.exs b/mix.exs index ca4c588..b1c633f 100644 --- a/mix.exs +++ b/mix.exs @@ -4,7 +4,7 @@ defmodule Pepper.HTTP.MixProject do def project do [ app: :pepper_http, - version: "0.7.0", + version: "0.9.0", elixir: "~> 1.12", start_permanent: Mix.env() == :prod, elixirc_options: [ diff --git a/test/pepper/http/unix_socket_options_test.exs b/test/pepper/http/unix_socket_options_test.exs new file mode 100644 index 0000000..e996234 --- /dev/null +++ b/test/pepper/http/unix_socket_options_test.exs @@ -0,0 +1,73 @@ +defmodule Pepper.HTTP.UnixSocketOptionsTest do + use ExUnit.Case, async: true + + import ExUnit.CaptureLog + + alias Pepper.HTTP.Client + alias Pepper.HTTP.ContentClient + + defmodule CaptureRequest do + def request(pid, request) do + send(pid, {:request, request}) + {:error, :captured} + end + end + + defp options(extra) do + [connection_manager: CaptureRequest, connection_manager_id: self()] ++ extra + end + + test "ContentClient forwards the socket option and keeps the HTTP destination" do + log = + capture_log(fn -> + assert {:error, :captured} = + ContentClient.get( + "http://service.invalid:8080/api/status", + [detail: "full"], + [], + options(unix_socket: "/tmp/service.sock") + ) + end) + + refute log =~ "unexpected option" + + assert_received {:request, request} + assert request.uri.host == "service.invalid" + assert request.uri.port == 8080 + assert request.path == "/api/status?detail=full" + assert request.options[:unix_socket] == "/tmp/service.sock" + end + + test "nil disables the socket option" do + assert {:error, :captured} = + Client.request( + :get, + "http://localhost/status", + [], + nil, + options(unix_socket: nil) + ) + + assert_received {:request, request} + assert request.options[:unix_socket] == nil + end + + test "rejects invalid socket paths before submitting a request" do + for path <- ["", 123, String.to_charlist("/tmp/service.sock"), "/tmp/service\0.sock"] do + assert_raise ArgumentError, ":unix_socket must be a non-empty path string or nil", fn -> + Client.request(:get, "http://localhost/status", [], nil, options(unix_socket: path)) + end + end + + refute_received {:request, _} + end + + test "rejects proxy options with a Unix socket before connecting" do + assert_raise ArgumentError, ":unix_socket cannot be combined with a proxy", fn -> + Client.request(:get, "http://localhost/status", [], nil, + unix_socket: "/tmp/service.sock", + connect_options: [proxy: {:http, "localhost", 8080, []}] + ) + end + end +end diff --git a/test/pepper/http/unix_socket_test.exs b/test/pepper/http/unix_socket_test.exs new file mode 100644 index 0000000..6618272 --- /dev/null +++ b/test/pepper/http/unix_socket_test.exs @@ -0,0 +1,151 @@ +defmodule Pepper.HTTP.UnixSocketTest do + use ExUnit.Case, async: true + + alias Pepper.HTTP.Client + alias Pepper.HTTP.ContentClient + alias Pepper.HTTP.ConnectionManager.Pooled + + @moduletag :unix_socket + @moduletag skip: not match?({:unix, _}, :os.type()) + + defmodule Server do + import Plug.Conn + + def init(options), do: options + + def call(conn, {parent, identity}) do + {:ok, body, conn} = read_body(conn) + {_, request} = conn.adapter + + send( + parent, + {:served, identity, request.pid, conn.method, conn.request_path, conn.query_string, + conn.host, conn.port, conn.req_headers, body} + ) + + conn |> put_resp_content_type("text/plain") |> send_resp(200, identity) + end + end + + defp start_server(identity, transport) do + ref = make_ref() + options = [ref: ref, port: 0, transport_options: [num_acceptors: 2]] ++ transport + + child = + Plug.Cowboy.child_spec(scheme: :http, plug: {Server, {self(), identity}}, options: options) + + start_supervised!(Supervisor.child_spec(child, id: ref)) + ref + end + + defp start_socket(identity) do + path = Path.join(System.tmp_dir!(), "pepper-#{System.unique_integer([:positive])}.sock") + on_exit(fn -> File.rm(path) end) + start_server(identity, ip: {:local, path}) + path + end + + defp client_options(:one_off, protocol) do + [connect_options: [protocols: [protocol]]] + end + + defp client_options(:pooled, protocol) do + pool = start_supervised!({Pooled, [[pool_size: 1], []]}) + + [ + connection_manager: :pooled, + connection_manager_id: pool, + connect_options: [protocols: [protocol]] + ] + end + + for manager <- [:one_off, :pooled], protocol <- [:http1, :http2] do + test "#{manager} #{protocol} keeps the HTTP host, port, path, query, and body" do + socket = start_socket("unix") + options = client_options(unquote(manager), unquote(protocol)) ++ [unix_socket: socket] + + assert {:ok, %{status_code: 200, protocol: unquote(protocol)}, {:text, "unix"}} = + ContentClient.post( + "http://service.invalid:8080/api/status", + [detail: "full"], + [], + {:text, "hello"}, + options + ) + + assert_receive {:served, "unix", _, "POST", "/api/status", "detail=full", "service.invalid", + 8080, _, "hello"} + + assert {:ok, %{body: "unix"}} = + Client.request(:get, "http://service.invalid/status", [], nil, options) + + assert_receive {:served, "unix", _, "GET", "/status", "", "service.invalid", 80, _, ""} + end + end + + for manager <- [:one_off, :pooled] do + test "#{manager} preserves an explicit Host header" do + socket = start_socket("unix") + options = client_options(unquote(manager), :http1) ++ [unix_socket: socket] + + assert {:ok, %{body: "unix"}} = + Client.request( + :get, + "http://service.invalid/status", + [{"Host", "virtual.example:9090"}], + nil, + options + ) + + assert_receive {:served, "unix", _, "GET", "/status", "", "virtual.example", 9090, _, ""} + end + end + + for protocol <- [:http1, :http2] do + test "#{protocol} reuses a socket and reconnects a reclaimed worker across sockets and TCP" do + socket_a = start_socket("a") + socket_b = start_socket("b") + tcp_ref = start_server("tcp", ip: {127, 0, 0, 1}) + url = "http://localhost:#{:ranch.get_port(tcp_ref)}/status" + options = client_options(:pooled, unquote(protocol)) + + request = fn identity, socket_options -> + assert {:ok, %{body: ^identity}} = + Client.request(:get, url, [], nil, options ++ socket_options) + + assert_receive {:served, ^identity, connection, "GET", "/status", "", _, _, _, ""} + await_checkin(options[:connection_manager_id], 100) + connection + end + + first = request.("a", unix_socket: socket_a) + assert first == request.("a", unix_socket: socket_a) + assert first != request.("b", unix_socket: socket_b) + request.("tcp", []) + assert first != request.("a", unix_socket: socket_a) + end + end + + defp await_checkin(pool, attempts) do + case Pooled.get_stats(pool) do + %{busy_size: 0} -> + :ok + + _ when attempts > 0 -> + Process.sleep(1) + await_checkin(pool, attempts - 1) + + stats -> + flunk("connection was not returned to pool: #{inspect(stats)}") + end + end + + test "connect errors retain the socket option in the request" do + path = Path.join(System.tmp_dir!(), "pepper-missing-#{System.unique_integer([:positive])}.sock") + + assert {:error, %Pepper.HTTP.ConnectError{request: request, reason: %Mint.TransportError{}}} = + Client.request(:get, "http://service.invalid/status", [], nil, unix_socket: path) + + assert request.options[:unix_socket] == path + end +end