feat: add connect_to option (#183)

* feat: add connect_to option

Add a `connect_to` request option that lets
ibrowse connect to a different network target
while preserving the original URL host for Host
handling, cookies, and TLS SNI.

The use case for this is when connecting through
a transparent SNI proxy for egress control.

* Add connect_to tests

These are for Will's "feat: add connect_to option" feature

---------

Co-authored-by: Will Holley <will.holley@uk.ibm.com>

Author: nickva

src/ibrowse_http_client.erl

@@ -580,13 +580,18 @@ do_connect(Host, Port, Options, #state{is_ssl      = true,
                                        use_proxy   = false,
                                        ssl_options = SSLOptions},
            Timeout) ->
+    %% Check for connect_to override and remove from options
+    Host1 = get_value(connect_to, Options, Host),
+    Options1 = proplists:delete(connect_to, Options),
+    
     %% if a socks5 proxy is configured, open the socket separately
     %% before upgrading the socket to a TLS connection.
-    case get_value(socks5_host, Options, undefined) of
+    case get_value(socks5_host, Options1, undefined) of
         %% no socks5 proxy is configured, connect directly with TLS:
         undefined ->
-            Sock_options = get_sock_options(Host, Options, SSLOptions),
-            ssl:connect(Host, Port, Sock_options, Timeout);
+            Sock_options = get_sock_options(Host, Options1, SSLOptions),
+            Sock_options1 = ensure_sni(Sock_options, Host),
+            ssl:connect(Host1, Port, Sock_options1, Timeout);
 
         %% proxy configuration is present: first establish a socket
         %% and then upgrade:
@@ -603,13 +608,17 @@ do_connect(Host, Port, Options, #state{is_ssl      = true,
     end;
 
 do_connect(Host, Port, Options, _State, Timeout) ->
-    Socks5Host = get_value(socks5_host, Options, undefined),
-    Sock_options = get_sock_options(Host, Options, []),
+    %% Check for connect_to override and remove from options
+    Host1 = get_value(connect_to, Options, Host),
+    Options1 = proplists:delete(connect_to, Options),
+    
+    Socks5Host = get_value(socks5_host, Options1, undefined),
+    Sock_options = get_sock_options(Host, Options1, []),
     case Socks5Host of
       undefined ->
-        gen_tcp:connect(Host, Port, Sock_options, Timeout);
+        gen_tcp:connect(Host1, Port, Sock_options, Timeout);
       _ ->
-        catch ibrowse_socks5:connect(Host, Port, Options, Sock_options, Timeout)
+        catch ibrowse_socks5:connect(Host1, Port, Options1, Sock_options, Timeout)
     end.
 
 get_sock_options(Host, Options, SSLOptions) ->
@@ -657,10 +666,20 @@ filter_sock_options(Opts) ->
                          false;
                     (list) ->
                          false;
+
                     (_) ->
                          true
                  end, Opts).
 
+%% Ensure SNI is set for SSL connections when using connect_to override
+ensure_sni(Opts, Host) ->
+    case lists:keyfind(server_name_indication, 1, Opts) of
+        false ->
+            [{server_name_indication, Host} | Opts];
+        _ ->
+            Opts
+    end.
+
 do_send(Req, #state{socket = Sock,
                     is_ssl = true,
                     use_proxy = true,

test/ibrowse_test.erl

@@ -34,6 +34,9 @@
          test_preserve_status_line/0,
          test_binary_headers/0,
          test_binary_headers/1,
+         test_connect_to_overrides_target/0,
+         test_connect_to_unreachable_fails/0,
+         test_connect_to_preserves_host_header/0,
          test_dead_lb_pid/0,
          test_generate_body_0/0,
          test_retry_of_requests/0,
@@ -62,6 +65,9 @@
                       {local_test_fun, test_303_response_with_a_body, []},
 		      {local_test_fun, test_303_response_with_no_body, []},
                       {local_test_fun, test_binary_headers, []},
+                      {local_test_fun, test_connect_to_overrides_target, []},
+                      {local_test_fun, test_connect_to_unreachable_fails, []},
+                      {local_test_fun, test_connect_to_preserves_host_header, []},
                       {local_test_fun, test_dead_lb_pid, []},
                       {local_test_fun, test_retry_of_requests, []},
 		      {local_test_fun, verify_chunked_streaming, []},
@@ -546,6 +552,55 @@ test_binary_headers(Url) ->
             {test_failed, Res}
     end.
 
+%%------------------------------------------------------------------------------
+%% url host is bogus, request only succeeds if connect_to actually
+%% redirects
+%%------------------------------------------------------------------------------
+test_connect_to_overrides_target() ->
+    clear_msg_q(),
+    Url = "http://no.such.host.invalid:8181/",
+    case ibrowse:send_req(Url, [], get, [], [{connect_to, "localhost"}], 5000) of
+        {ok, "200", _, _} ->
+            success;
+        Res ->
+            {test_failed, Res}
+    end.
+
+%%------------------------------------------------------------------------------
+%% url host reachable, but connect_to points at an unroutable
+%% address
+%%------------------------------------------------------------------------------
+test_connect_to_unreachable_fails() ->
+    clear_msg_q(),
+    case ibrowse:send_req("http://localhost:8181/", [], get, [],
+                          [{connect_to, "192.0.2.1"},
+                           {connect_timeout, 500}],
+                          5000) of
+        {error, _} ->
+            success;
+        Res ->
+            {test_failed, Res}
+    end.
+
+%%------------------------------------------------------------------------------
+%% connect_to shouldn't bleed into the Host header, server should see the
+%% original url host
+%%------------------------------------------------------------------------------
+test_connect_to_preserves_host_header() ->
+    clear_msg_q(),
+    Url = "http://example.invalid:8181/ibrowse_echo_host",
+    case ibrowse:send_req(Url, [], get, [], [{connect_to, "localhost"}], 5000) of
+        {ok, "200", Headers, _} ->
+            case proplists:get_value("x-host", Headers) of
+                "example.invalid:8181" ->
+                    success;
+                V ->
+                    {fail, V}
+            end;
+        Res ->
+            {test_failed, Res}
+    end.
+
 %%------------------------------------------------------------------------------
 %% Test what happens when the response to a HEAD request is a
 %% Chunked-Encoding response with a non-empty body. Issue #67 on

test/ibrowse_test_server.erl

@@ -229,6 +229,16 @@ process_request(Sock, Sock_type,
 	false ->
 	    collect_body
     end;
+process_request(Sock, Sock_type,
+                #request{method='GET',
+                         headers = Headers,
+                         uri = {abs_path, "/ibrowse_echo_host"}}) ->
+    Host = get_host_header(Headers),
+    Resp = [<<"HTTP/1.1 200 OK\r\n">>,
+            <<"Server: ibrowse_test\r\n">>,
+            "x-host: ", Host, "\r\n",
+            <<"Content-Length: 0\r\n\r\n">>],
+    do_send(Sock, Sock_type, Resp);
 process_request(Sock, Sock_type,
                 #request{method='GET',
                          headers = Headers,
@@ -366,3 +376,10 @@ get_content_length([{http_header, _, _X, _, _Y} | T]) ->
     get_content_length(T);
 get_content_length([]) ->
     undefined.
+
+get_host_header([{http_header, _, 'Host', _, V} | _]) ->
+    V;
+get_host_header([_ | T]) ->
+    get_host_header(T);
+get_host_header([]) ->
+    "not_found".