fix(security): add dial-time SSRF guard for participant InferenceUrl (#1470)

Registration-time validation of participant InferenceUrl only rejects
literal private IPs and localhost strings; it never resolves DNS, because
ValidateBasic must stay deterministic for consensus. A hostname therefore
always passes, and DNS rebinding needs no new on-chain tx. Honest nodes
then dial that URL during mandatory payload retrieval and devshard peer
communication, so the dial can land on loopback, link-local metadata
(169.254.169.254), or RFC1918.

Fix at dial time, where the resolved IP is known. net.Dialer.Control runs
after resolution and once per candidate IP, with the address already
"ip:port", so it vets every real dial target: each dual-stack candidate
and each redirect hop. That is what defeats rebinding, which a
resolve-then-connect check cannot.

One guard (common/httpguard), reusing the registration gate's predicate
(utils.IsPrivateIP) so both agree on what "private" means. Two choke
points:

- common/validation.PayloadRetrievalClient: the default client for
  executor payload fetches, which also refuses redirects. Covers every
  FetchPayloadsHTTP caller.
- devshard/transport.getTransport: gateway/host peer dials, whose baseURL
  is the on-chain Participant.InferenceUrl.

Clients that dial our own infrastructure are deliberately left unguarded:
the devshardd inference engine's ML-node client, and devshardctl's chain
RPC / public-API clients. Those targets come from local config and
legitimately live on localhost/private ranges.

Local dev, docker-compose, and e2e register docker hostnames that resolve
to private IPs, so an opt-out is required. Secure by default: the guard is
active unless DEVSHARD_ALLOW_PRIVATE_ADDRESSES is set, which is enabled
only in local-test-net, testenv (gencompose), and testermint. Production
templates leave it unset.

Also fixes an IPv4-mapped IPv6 gap in the existing predicate, so
::ffff:169.254.169.254 is now classified private.

Author: Ryanchen911

common/httpguard/httpguard.go

@@ -0,0 +1,101 @@
+// Package httpguard provides a dial-time SSRF guard for HTTP clients that
+// connect to participant-controlled URLs taken from chain state (devshard peer
+// host URLs, executor payload endpoints derived from Participant.InferenceUrl).
+//
+// The guard hooks net.Dialer.Control, which the stdlib calls after DNS
+// resolution and once per candidate IP, with address already "ip:port".
+// Checking there vets every real dial target: each dual-stack candidate and
+// each redirect hop's fresh dial. That is what defeats DNS rebinding, which a
+// resolve-then-connect check cannot -- the gap between the lookup and the
+// connect is exactly the attack.
+//
+// The on-chain registration gate (inference-chain x/inference/utils
+// ValidateURLWithSSRFProtection) can only reject literal private IPs: it runs
+// inside a stateless ValidateBasic, so it must stay deterministic and cannot
+// resolve DNS. This guard is the enforcement point, and it reuses that gate's
+// predicate (utils.IsPrivateIP) so both agree on what "private" means.
+package httpguard
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"sync/atomic"
+	"syscall"
+	"time"
+
+	"github.com/productscience/inference/x/inference/utils"
+)
+
+// allowPrivate toggles the guard process-wide. Default false = secure.
+//
+// It is package-level rather than per-client because devshard's transport cache
+// is global and keyed only by baseURL, so a per-client setting could not be
+// threaded into an already-cached dialer. DialControl reads it on every dial,
+// so the toggle applies to clients constructed before it is set (notably the
+// package-level validation.PayloadRetrievalClient).
+var allowPrivate atomic.Bool
+
+// SetAllowPrivate configures whether guarded dialers may connect to
+// private/internal addresses. Callers wire this once at startup from their
+// environment (devshardd reads DEVSHARD_ALLOW_PRIVATE_ADDRESSES). Set true only
+// in local dev / docker-compose / e2e, where hosts register docker-internal
+// hostnames that resolve to private IPs.
+func SetAllowPrivate(allow bool) {
+	allowPrivate.Store(allow)
+}
+
+// AllowPrivate reports whether the guard is currently disabled.
+func AllowPrivate() bool {
+	return allowPrivate.Load()
+}
+
+// DialControl is a net.Dialer.Control hook that rejects connections to
+// private/internal IP addresses unless SetAllowPrivate(true) was called.
+func DialControl(_, address string, _ syscall.RawConn) error {
+	if allowPrivate.Load() {
+		return nil
+	}
+	host, _, err := net.SplitHostPort(address)
+	if err != nil {
+		// address is already "ip:port" at this point; fail closed.
+		return fmt.Errorf("ssrf guard: cannot parse dial address %q: %w", address, err)
+	}
+	ip := net.ParseIP(host)
+	if ip == nil {
+		return fmt.Errorf("ssrf guard: unresolved dial address %q", address)
+	}
+	if utils.IsPrivateIP(ip) {
+		return fmt.Errorf("ssrf guard: blocked dial to private address %s", ip)
+	}
+	return nil
+}
+
+// NewDialer returns a dialer carrying the guard, with the same timeouts the
+// stdlib default transport uses.
+func NewDialer() *net.Dialer {
+	return &net.Dialer{
+		Timeout:   30 * time.Second,
+		KeepAlive: 30 * time.Second,
+		Control:   DialControl,
+	}
+}
+
+// NewNoRedirectClient builds a guarded *http.Client that also refuses to follow
+// 3xx responses, so a public host cannot redirect a fetch to a private target.
+// The caller observes the 3xx as a non-200 status. Redirect refusal is defense
+// in depth: DialControl already re-checks each hop's dial.
+//
+// The transport is cloned from http.DefaultTransport so proxy/keep-alive/HTTP2
+// behavior matches the stdlib.
+func NewNoRedirectClient(timeout time.Duration) *http.Client {
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	transport.DialContext = NewDialer().DialContext
+	return &http.Client{
+		Timeout:   timeout,
+		Transport: transport,
+		CheckRedirect: func(*http.Request, []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+}

common/httpguard/httpguard_test.go

@@ -0,0 +1,178 @@
+package httpguard
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+)
+
+// resolveTo builds a client whose dialer is the guarded one but whose DNS
+// answers are forced to target, simulating a participant who registered a
+// hostname resolving (or rebinding) to an address of their choosing.
+func resolveTo(t *testing.T, target string) *http.Client {
+	t.Helper()
+	client := NewNoRedirectClient(5 * time.Second)
+	transport := client.Transport.(*http.Transport)
+	dialer := NewDialer()
+	transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
+		_, port, err := net.SplitHostPort(addr)
+		if err != nil {
+			return nil, err
+		}
+		return dialer.DialContext(ctx, network, net.JoinHostPort(target, port))
+	}
+	return client
+}
+
+func requireBlocked(t *testing.T, err error) {
+	t.Helper()
+	if err == nil {
+		t.Fatal("expected the dial to be blocked, got nil error")
+	}
+	if !strings.Contains(err.Error(), "ssrf guard") {
+		t.Fatalf("expected an ssrf guard error, got %v", err)
+	}
+}
+
+func TestDialControlBlocksPrivateTargets(t *testing.T) {
+	SetAllowPrivate(false)
+
+	// Each case is a hostname that resolves to a private target: the literal
+	// forms an attacker can register plus the ones a naive string check misses.
+	cases := map[string]string{
+		"loopback":            "127.0.0.1",
+		"loopback_upper_8":    "127.5.6.7",
+		"cloud_metadata":      "169.254.169.254",
+		"rfc1918_10":          "10.0.0.1",
+		"rfc1918_172":         "172.16.0.1",
+		"rfc1918_192":         "192.168.1.1",
+		"unspecified":         "0.0.0.0",
+		"ipv6_loopback":       "::1",
+		"ipv6_link_local":     "fe80::1",
+		"ipv6_ula":            "fc00::1",
+		"ipv4_mapped_v6":      "::ffff:10.0.0.1",
+		"ipv4_mapped_meta_v6": "::ffff:169.254.169.254",
+	}
+
+	for name, target := range cases {
+		t.Run(name, func(t *testing.T) {
+			client := resolveTo(t, target)
+			_, err := client.Get("http://ssrf.attacker.tld/")
+			requireBlocked(t, err)
+		})
+	}
+}
+
+// A hostname that resolves to loopback is the core of issue #1470: the on-chain
+// registration gate cannot reject it (no DNS in ValidateBasic), so the dial is
+// where it must fail. Uses a real server to prove the request never lands.
+func TestGuardBlocksHostnameResolvingToLoopback(t *testing.T) {
+	SetAllowPrivate(false)
+
+	var reached bool
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		reached = true
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(server.Close)
+
+	_, port, err := net.SplitHostPort(strings.TrimPrefix(server.URL, "http://"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	client := resolveTo(t, "127.0.0.1")
+	_, err = client.Get("http://ssrf.attacker.tld:" + port + "/")
+	requireBlocked(t, err)
+	if reached {
+		t.Fatal("guard let the request through to the loopback server")
+	}
+}
+
+// Decimal and hex host forms are alternate spellings of 127.0.0.1. The guard
+// checks the resolved IP, so the spelling is irrelevant -- this pins that.
+func TestGuardBlocksNumericLoopbackSpellings(t *testing.T) {
+	SetAllowPrivate(false)
+
+	for _, raw := range []string{"http://2130706433/", "http://0x7f000001/", "http://127.1/"} {
+		t.Run(raw, func(t *testing.T) {
+			client := NewNoRedirectClient(5 * time.Second)
+			_, err := client.Get(raw)
+			requireBlocked(t, err)
+		})
+	}
+}
+
+// A public host answering 302 -> 127.0.0.1 must not reach the private target.
+// The redirect is refused outright; had it been followed, the new hop's dial
+// would hit the guard too.
+func TestNoRedirectClientDoesNotFollowRedirectToPrivate(t *testing.T) {
+	SetAllowPrivate(true) // let the public-side server on loopback be reachable
+
+	var privateReached bool
+	private := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		privateReached = true
+	}))
+	t.Cleanup(private.Close)
+
+	public := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, private.URL, http.StatusFound)
+	}))
+	t.Cleanup(public.Close)
+
+	client := NewNoRedirectClient(5 * time.Second)
+	resp, err := client.Get(public.URL)
+	if err != nil {
+		t.Fatalf("expected the 3xx to surface as a response, got %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusFound {
+		t.Fatalf("expected the caller to observe 302, got %d", resp.StatusCode)
+	}
+	if privateReached {
+		t.Fatal("client followed the redirect into the private target")
+	}
+}
+
+// Dev/test environments register docker-internal hostnames that resolve to
+// private IPs, so the opt-out has to actually let them through.
+func TestAllowPrivateLetsPrivateTargetsThrough(t *testing.T) {
+	SetAllowPrivate(true)
+	t.Cleanup(func() { SetAllowPrivate(false) })
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+	}))
+	t.Cleanup(server.Close)
+
+	client := NewNoRedirectClient(5 * time.Second)
+	resp, err := client.Get(server.URL)
+	if err != nil {
+		t.Fatalf("allowPrivate should permit the loopback dial, got %v", err)
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("expected 200, got %d", resp.StatusCode)
+	}
+}
+
+func TestPublicAddressAllowedInBothModes(t *testing.T) {
+	for _, allow := range []bool{false, true} {
+		SetAllowPrivate(allow)
+		if err := DialControl("tcp", "93.184.216.34:80", nil); err != nil {
+			t.Fatalf("public address rejected with allowPrivate=%v: %v", allow, err)
+		}
+	}
+	SetAllowPrivate(false)
+}
+
+func TestDialControlFailsClosedOnMalformedAddress(t *testing.T) {
+	SetAllowPrivate(false)
+	requireBlocked(t, DialControl("tcp", "not-an-address", nil))
+	requireBlocked(t, DialControl("tcp", "still.a.hostname:80", nil))
+}

common/validation/payload_retrieval.go

@@ -2,6 +2,7 @@ package validation
 
 import (
 	"common/completionapi"
+	"common/httpguard"
 	"common/logging"
 	"common/utils"
 	"context"
@@ -34,9 +35,13 @@ var ErrEpochStale = errors.New("inference epoch too old, validation no longer us
 var ErrPayloadGone = errors.New("payload no longer available on executor")
 
 // PayloadRetrievalClient is the default HTTP client for payload retrieval.
-var PayloadRetrievalClient = &http.Client{
-	Timeout: 30 * time.Second,
-}
+//
+// The request URL is built from the executor's on-chain InferenceUrl, which the
+// executor controls, so this client carries the dial-time SSRF guard and refuses
+// redirects. Without it a participant could register a hostname that resolves
+// (or later rebinds) to loopback/RFC1918/cloud-metadata and make every validator
+// fetching its payloads connect there. See common/httpguard.
+var PayloadRetrievalClient = httpguard.NewNoRedirectClient(30 * time.Second)
 
 // PayloadResponse matches the executor endpoint response.
 // Used by both chain validation and devshard validation paths.

devshard/cmd/devshardctl/main.go

@@ -15,6 +15,7 @@ import (
 	"time"
 
 	"common/chain"
+	"common/httpguard"
 	"devshard/bridge"
 	"devshard/state"
 	"devshard/types"
@@ -128,6 +129,11 @@ var gatewayRuntimeBuilder = buildRuntime
 func main() {
 	ConfigurePoCRequestMode(os.Getenv("DEVSHARD_POC_REQUEST_MODE"))
 	ConfigureCapacityAwareLimits(os.Getenv("DEVSHARD_CAPACITY_AWARE_LIMITS"))
+	// Wire the dial-time SSRF guard before any outbound dial. Host URLs come
+	// from chain state and are participant-controlled; the gateway's own chain
+	// RPC/public-API clients are unguarded, so private self-hosted endpoints
+	// keep working. Default secure; dev/e2e opt out via env.
+	httpguard.SetAllowPrivate(readBoolEnv("DEVSHARD_ALLOW_PRIVATE_ADDRESSES", false))
 	flags := parseCLIFlags()
 	runtimeOpts := mustLoadRuntimeOptions(flags)
 	gatewayStore := mustOpenGatewayStore(runtimeOpts.baseStorageDir)

devshard/cmd/devshardd/app.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"common/chain"
+	"common/httpguard"
 	mlnodeclient "common/nodemanager"
 	commrc "common/runtimeconfig"
 	"common/storage/payloads"
@@ -77,6 +78,15 @@ func buildApp(ctx context.Context, cfg runtimeConfig) (_ *devshardApp, err error
 		return nil, fmt.Errorf("create data dir %s: %w", cfg.DataDir, err)
 	}
 
+	// Wire the dial-time SSRF guard before anything can dial out. Guarded
+	// clients read the flag per dial, so this also covers the package-level
+	// validation.PayloadRetrievalClient constructed at init.
+	httpguard.SetAllowPrivate(cfg.AllowPrivateAddresses)
+	if cfg.AllowPrivateAddresses {
+		slog.Warn("SSRF guard disabled: dials to private/internal addresses are allowed",
+			"env", "DEVSHARD_ALLOW_PRIVATE_ADDRESSES")
+	}
+
 	var closers closeStack
 	defer func() {
 		if err != nil {

devshard/cmd/devshardd/config.go

@@ -26,14 +26,20 @@ import (
 var sdkConfigOnce sync.Once
 
 type runtimeConfig struct {
-	Port                    int
-	AdminAddr               string
-	DataDir                 string
-	BinaryLogVersion        string
-	RuntimeVersion          string
-	ProtocolVersion         string
-	NodeManagerAddr         string
-	HostEventsEnabled       bool
+	Port              int
+	AdminAddr         string
+	DataDir           string
+	BinaryLogVersion  string
+	RuntimeVersion    string
+	ProtocolVersion   string
+	NodeManagerAddr   string
+	HostEventsEnabled bool
+	// AllowPrivateAddresses disables the dial-time SSRF guard on outbound
+	// connections to participant-controlled URLs (peer devshard hosts, executor
+	// payload endpoints). Default false = secure. Set true only in local dev /
+	// docker-compose / e2e, where hosts register docker-internal hostnames that
+	// resolve to private IPs. Env: DEVSHARD_ALLOW_PRIVATE_ADDRESSES.
+	AllowPrivateAddresses   bool
 	ValidationRetryInterval time.Duration
 	ValidationLeaseTTL      time.Duration
 	ShutdownGrace           time.Duration
@@ -146,6 +152,7 @@ func loadRuntimeConfig(args []string, protocolVersion, linkBinaryVersion string)
 		ProtocolVersion:         protocolVersion,
 		NodeManagerAddr:         envOr("NODE_MANAGER_ADDR", "localhost:9400"),
 		HostEventsEnabled:       envBoolOr("DEVSHARD_HOST_EVENTS_ENABLED", true),
+		AllowPrivateAddresses:   envBoolOr("DEVSHARD_ALLOW_PRIVATE_ADDRESSES", false),
 		ValidationRetryInterval: retryInterval,
 		ValidationLeaseTTL:      leaseTTL,
 		ShutdownGrace:           shutdownGrace,