Block SSRF to non-public addresses via connect-time IP guard

README.md

@@ -19,8 +19,11 @@
 | cf-account-id| CF_ACCOUNT_ID   | none           | Cloudflare account ID for Browser Rendering API       |
 | cf-api-token | CF_API_TOKEN    | none           | Cloudflare API token with Browser Rendering Edit perm |
 | cf-route-all | CF_ROUTE_ALL    | `false`        | route every request through Cloudflare Browser Rendering |
+| allow-private-networks | ALLOW_PRIVATE_NETWORKS | `false` | allow fetching URLs that resolve to loopback/private/link-local addresses (disables the SSRF guard; enable only for trusted intranet use) |
 | dbg          | DEBUG           | `false`        | debug mode                                            |
 
+The SSRF guard (on by default) blocks fetches to non-public addresses and, to prevent bypass, ignores `HTTP_PROXY`/`HTTPS_PROXY` on outbound requests. Deployments that must reach the internet through an outbound proxy should run with `--allow-private-networks` (which restores proxy-aware transport) and enforce egress restrictions at the network layer instead. Note that requests routed through Cloudflare Browser Rendering are validated on the submitted host only; Cloudflare performs its own DNS resolution and redirect following remotely, so it is not covered by the connect-time guard (it cannot reach this service's internal network either way).
+
 ### Cloudflare Browser Rendering (optional)
 
 Cloudflare Browser Rendering is useful for JavaScript-heavy pages and sites behind a "please enable JS" wall, but it's slower than direct HTTP and the free tier throttles at 1 request per 10 seconds. To keep the service cost-effective, Cloudflare routing is **opt-in**.

extractor/pics.go

@@ -11,6 +11,19 @@ import (
 	log "github.com/go-pkgz/lgr"
 )
 
+// imageClient returns a lazily-built HTTP client for image size probes, shared across all fetches
+// (a Transport is long-lived and pools connections; building one per image would churn FDs), and
+// honoring BlockPrivateNetworks to guard against SSRF via image URLs.
+func (f *UReadability) imageClient() *http.Client {
+	f.imgClientOnce.Do(func() {
+		f.imgClient = &http.Client{Timeout: 30 * time.Second}
+		if f.BlockPrivateNetworks {
+			f.imgClient.Transport = safeTransport(30 * time.Second)
+		}
+	})
+	return f.imgClient
+}
+
 func (f *UReadability) extractPics(iselect *goquery.Selection, url string) (mainImage string, allImages []string, ok bool) {
 	images := make(map[int]string)
 
@@ -57,15 +70,13 @@ func (f *UReadability) extractPics(iselect *goquery.Selection, url string) (main
 
 // getImageSize loads image to get size
 func (f *UReadability) getImageSize(url string) (size int) {
-	httpClient := &http.Client{Timeout: 30 * time.Second}
 	req, err := http.NewRequest("GET", url, http.NoBody)
 	if err != nil {
 		log.Printf("[WARN] can't create request to get pic from %s", url)
 		return 0
 	}
-	req.Close = true
 	req.Header.Set("User-Agent", userAgent)
-	resp, err := httpClient.Do(req)
+	resp, err := f.imageClient().Do(req)
 	if err != nil {
 		log.Printf("[WARN] can't get %s, error=%v", url, err)
 		return 0

extractor/pics_test.go

@@ -92,3 +92,21 @@ func TestExtractPicsDirectly(t *testing.T) {
 		assert.Equal(t, ts.URL, im)
 	})
 }
+
+func TestGetImageSize_BlockPrivateNetworks(t *testing.T) {
+	payload := strings.Repeat("x", 512)
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte(payload))
+	}))
+	defer ts.Close()
+
+	t.Run("blocked loopback yields zero size", func(t *testing.T) {
+		lr := UReadability{TimeOut: 5 * time.Second, BlockPrivateNetworks: true}
+		assert.Equal(t, 0, lr.getImageSize(ts.URL))
+	})
+
+	t.Run("allowed when disabled", func(t *testing.T) {
+		lr := UReadability{TimeOut: 5 * time.Second}
+		assert.Equal(t, len(payload), lr.getImageSize(ts.URL))
+	})
+}

extractor/readability.go

@@ -4,6 +4,7 @@ package extractor
 import (
 	"context"
 	"fmt"
+	"net/http"
 	"net/url"
 	"regexp"
 	"strings"
@@ -37,9 +38,15 @@ type UReadability struct {
 	Retriever   Retriever // default retriever; when nil a cached HTTPRetriever is used
 	CFRetriever Retriever // optional Cloudflare Browser Rendering retriever; when set, enables routing
 	CFRouteAll  bool      // route every request through CFRetriever (requires CFRetriever != nil)
+	// BlockPrivateNetworks rejects image fetches to non-public addresses. Guards image extraction
+	// against SSRF; set together with HTTPRetriever.BlockPrivateNetworks in public deployments.
+	BlockPrivateNetworks bool
 
 	defaultRetrieverOnce sync.Once
 	defaultRetriever     Retriever
+
+	imgClientOnce sync.Once
+	imgClient     *http.Client
 }
 
 // retriever returns the configured default Retriever, creating a cached HTTPRetriever if nil
@@ -48,7 +55,7 @@ func (f *UReadability) retriever() Retriever {
 		return f.Retriever
 	}
 	f.defaultRetrieverOnce.Do(func() {
-		f.defaultRetriever = &HTTPRetriever{Timeout: f.TimeOut}
+		f.defaultRetriever = &HTTPRetriever{Timeout: f.TimeOut, BlockPrivateNetworks: f.BlockPrivateNetworks}
 	})
 	return f.defaultRetriever
 }
@@ -109,6 +116,18 @@ func (f *UReadability) extractWithRules(ctx context.Context, reqURL string, rule
 	log.Printf("[INFO] extract %s", reqURL)
 	rb := &Response{}
 
+	// enforce the SSRF policy before any retriever runs, so it also vets the submitted host on the
+	// Cloudflare path (which fetches remotely and isn't guarded by the connect-time dialer). the HTTP
+	// retriever additionally re-checks at connect time, defending against DNS rebinding and redirects
+	// between this lookup and the dial; the Cloudflare path can't be redirect-guarded from here, but
+	// it fetches on Cloudflare's infrastructure and so cannot reach this service's own internal network.
+	if f.BlockPrivateNetworks {
+		if err := validatePublicHost(ctx, reqURL); err != nil {
+			log.Printf("[WARN] blocked fetch of %s: %v", reqURL, err)
+			return nil, err
+		}
+	}
+
 	// look up a rule by domain once up front (unless one was explicitly passed) so retriever
 	// selection and getContent share the same lookup instead of paying for two round-trips.
 	if rule == nil && f.Rules != nil {

extractor/retriever.go

@@ -32,6 +32,9 @@ type RetrieveResult struct {
 // HTTPRetriever fetches pages using a standard HTTP client
 type HTTPRetriever struct {
 	Timeout time.Duration
+	// BlockPrivateNetworks rejects fetches that resolve to loopback, private, link-local or other
+	// non-public addresses. Guards against SSRF via user-supplied URLs; enable in public deployments.
+	BlockPrivateNetworks bool
 
 	once   sync.Once
 	client *http.Client
@@ -46,6 +49,9 @@ func (h *HTTPRetriever) httpClient() *http.Client {
 			timeout = httpDefaultTimeout
 		}
 		h.client = &http.Client{Timeout: timeout}
+		if h.BlockPrivateNetworks {
+			h.client.Transport = safeTransport(timeout)
+		}
 	})
 	return h.client
 }

extractor/retriever_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"io"
+	"net"
 	"net/http"
 	"net/http/httptest"
 	"sync/atomic"
@@ -81,6 +82,66 @@ func TestHTTPRetriever_Retrieve(t *testing.T) {
 	}
 }
 
+func TestHTTPRetriever_BlockPrivateNetworks(t *testing.T) {
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		_, _ = w.Write([]byte("<html><body>internal</body></html>"))
+	}))
+	defer ts.Close()
+
+	t.Run("blocks loopback when enabled", func(t *testing.T) {
+		retriever := &HTTPRetriever{Timeout: 5 * time.Second, BlockPrivateNetworks: true}
+		result, err := retriever.Retrieve(context.Background(), ts.URL)
+		require.Error(t, err)
+		assert.Nil(t, result)
+		assert.ErrorIs(t, err, errBlockedAddress)
+	})
+
+	t.Run("allows loopback when disabled", func(t *testing.T) {
+		retriever := &HTTPRetriever{Timeout: 5 * time.Second}
+		result, err := retriever.Retrieve(context.Background(), ts.URL)
+		require.NoError(t, err)
+		assert.Equal(t, "<html><body>internal</body></html>", string(result.Body))
+	})
+}
+
+func TestIsBlockedIP(t *testing.T) {
+	tests := []struct {
+		ip      string
+		blocked bool
+	}{
+		{"127.0.0.1", true},
+		{"::1", true},
+		{"10.0.0.1", true},
+		{"192.168.1.1", true},
+		{"172.16.0.1", true},
+		{"169.254.169.254", true}, // cloud metadata endpoint
+		{"0.0.0.0", true},
+		{"0.1.2.3", true},         // 0.0.0.0/8 this-network
+		{"100.64.0.1", true},      // CGNAT
+		{"198.18.0.5", true},      // benchmarking
+		{"203.0.113.7", true},     // documentation
+		{"240.0.0.1", true},       // reserved
+		{"255.255.255.255", true}, // broadcast
+		{"2001:db8::1", true},     // IPv6 documentation
+		{"8.8.8.8", false},
+		{"1.1.1.1", false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.ip, func(t *testing.T) {
+			assert.Equal(t, tt.blocked, isBlockedIP(net.ParseIP(tt.ip)))
+		})
+	}
+}
+
+func TestValidatePublicHost(t *testing.T) {
+	ctx := context.Background()
+	require.Error(t, validatePublicHost(ctx, "http://127.0.0.1/x"), "loopback literal blocked")
+	require.Error(t, validatePublicHost(ctx, "http://169.254.169.254/latest"), "metadata literal blocked")
+	require.Error(t, validatePublicHost(ctx, "http://localhost:8080/"), "localhost resolves to loopback")
+	require.Error(t, validatePublicHost(ctx, "http:///nohost"), "missing host blocked")
+	require.NoError(t, validatePublicHost(ctx, "http://8.8.8.8/"), "public literal allowed")
+}
+
 func TestHTTPRetriever_UserAgent(t *testing.T) {
 	var receivedUA string
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

extractor/safedial.go

@@ -0,0 +1,125 @@
+package extractor
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"syscall"
+	"time"
+)
+
+// errBlockedAddress is returned when a fetch targets a non-public address and private-network
+// blocking is enabled. the check runs at connect time against the actual resolved IP, so it also
+// defends against DNS rebinding and redirects that point back at internal hosts.
+var errBlockedAddress = errors.New("connection to non-public address blocked")
+
+// extraBlockedCIDRs are IANA special-purpose ranges that the net.IP helpers below do not classify
+// as private/loopback/link-local but which are still not public, routable destinations. Blocking
+// them closes SSRF paths into CGNAT, benchmarking, documentation, reserved and broadcast space.
+var extraBlockedCIDRs = parseCIDRs(
+	"0.0.0.0/8",          // RFC1122 "this network" (IsUnspecified only covers 0.0.0.0 itself)
+	"100.64.0.0/10",      // RFC6598 carrier-grade NAT
+	"192.0.0.0/24",       // RFC6890 IETF protocol assignments
+	"192.0.2.0/24",       // TEST-NET-1 documentation
+	"198.18.0.0/15",      // RFC2544 benchmarking
+	"198.51.100.0/24",    // TEST-NET-2 documentation
+	"203.0.113.0/24",     // TEST-NET-3 documentation
+	"240.0.0.0/4",        // RFC1112 reserved (former class E)
+	"255.255.255.255/32", // limited broadcast
+	"64:ff9b::/96",       // RFC6052 NAT64
+	"100::/64",           // RFC6666 discard-only
+	"2001:db8::/32",      // IPv6 documentation
+)
+
+func parseCIDRs(cidrs ...string) []*net.IPNet {
+	nets := make([]*net.IPNet, 0, len(cidrs))
+	for _, c := range cidrs {
+		if _, n, err := net.ParseCIDR(c); err == nil {
+			nets = append(nets, n)
+		}
+	}
+	return nets
+}
+
+// isBlockedIP reports whether ip is outside the public routable range and must not be dialed.
+func isBlockedIP(ip net.IP) bool {
+	if ip == nil {
+		return true
+	}
+	if ip.IsLoopback() || ip.IsPrivate() || ip.IsUnspecified() ||
+		ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsMulticast() {
+		return true
+	}
+	for _, n := range extraBlockedCIDRs {
+		if n.Contains(ip) {
+			return true
+		}
+	}
+	return false
+}
+
+// safeControl is a net.Dialer Control hook that rejects connections to non-public addresses.
+func safeControl(_, address string, _ syscall.RawConn) error {
+	host, _, err := net.SplitHostPort(address)
+	if err != nil {
+		return err
+	}
+	ip := net.ParseIP(host)
+	if ip == nil {
+		return fmt.Errorf("%w: cannot parse address %q", errBlockedAddress, host)
+	}
+	if isBlockedIP(ip) {
+		return fmt.Errorf("%w: %s", errBlockedAddress, ip)
+	}
+	return nil
+}
+
+// validatePublicHost rejects reqURL when its host is, or resolves to, a non-public address. It
+// enforces the SSRF policy uniformly across all retrievers, including Cloudflare Browser Rendering
+// which fetches remotely and so is not covered by the connect-time dialer guard.
+func validatePublicHost(ctx context.Context, reqURL string) error {
+	u, err := url.Parse(reqURL)
+	if err != nil {
+		return err
+	}
+	host := u.Hostname()
+	if host == "" {
+		return fmt.Errorf("%w: no host in %q", errBlockedAddress, reqURL)
+	}
+	if ip := net.ParseIP(host); ip != nil {
+		if isBlockedIP(ip) {
+			return fmt.Errorf("%w: %s", errBlockedAddress, ip)
+		}
+		return nil
+	}
+	ips, err := net.DefaultResolver.LookupIPAddr(ctx, host)
+	if err != nil {
+		return fmt.Errorf("resolve %q: %w", host, err)
+	}
+	for _, ipa := range ips {
+		if isBlockedIP(ipa.IP) {
+			return fmt.Errorf("%w: %s resolves to %s", errBlockedAddress, host, ipa.IP)
+		}
+	}
+	return nil
+}
+
+// safeTransport returns an http.Transport whose dialer blocks non-public addresses. Proxy support
+// is disabled: an env proxy would make the dialer validate the proxy's IP instead of the real
+// destination, defeating the guard. As a consequence, deployments that require HTTP_PROXY/HTTPS_PROXY
+// for outbound access must run with --allow-private-networks (which keeps the default proxy-aware
+// transport) and rely on network-level egress controls instead.
+func safeTransport(timeout time.Duration) *http.Transport {
+	dialer := &net.Dialer{Timeout: timeout, Control: safeControl}
+	tr, ok := http.DefaultTransport.(*http.Transport)
+	if !ok {
+		return &http.Transport{DialContext: dialer.DialContext, Proxy: nil}
+	}
+	cloned := tr.Clone()
+	cloned.Proxy = nil
+	cloned.DialContext = dialer.DialContext
+	return cloned
+}

main.go

@@ -30,7 +30,9 @@ var opts struct {
 	CFAccountID string            `long:"cf-account-id" env:"CF_ACCOUNT_ID" description:"Cloudflare account ID for Browser Rendering API"`
 	CFAPIToken  string            `long:"cf-api-token" env:"CF_API_TOKEN" description:"Cloudflare API token with Browser Rendering Edit permission"`
 	CFRouteAll  bool              `long:"cf-route-all" env:"CF_ROUTE_ALL" description:"route every request through Cloudflare Browser Rendering (requires cf-account-id and cf-api-token)"`
-	Debug       bool              `long:"dbg" env:"DEBUG" description:"debug mode"`
+	//nolint:lll // struct tag with env and description
+	AllowPrivateNetworks bool `long:"allow-private-networks" env:"ALLOW_PRIVATE_NETWORKS" description:"allow fetching URLs that resolve to loopback/private/link-local addresses (disables SSRF guard; enable only for trusted intranet use)"`
+	Debug                bool `long:"dbg" env:"DEBUG" description:"debug mode"`
 }
 
 func main() {
@@ -51,9 +53,15 @@ func main() {
 	}
 	stores := db.GetStores()
 
+	// block SSRF to non-public addresses by default; operators can opt out for trusted intranet use.
+	blockPrivate := !opts.AllowPrivateNetworks
+	if opts.AllowPrivateNetworks {
+		log.Print("[WARN] private-network SSRF guard disabled via --allow-private-networks")
+	}
+
 	// default retriever is always HTTP; CF is optional and, when configured, acts as a
 	// second retriever available for per-rule routing or global route-all.
-	httpRetriever := &extractor.HTTPRetriever{Timeout: 30 * time.Second}
+	httpRetriever := &extractor.HTTPRetriever{Timeout: 30 * time.Second, BlockPrivateNetworks: blockPrivate}
 	var cfRetriever extractor.Retriever
 	if opts.CFAccountID != "" && opts.CFAPIToken != "" {
 		cfRetriever = &extractor.CloudflareRetriever{
@@ -79,12 +87,13 @@ func main() {
 
 	srv := rest.Server{
 		Readability: extractor.UReadability{
-			TimeOut:     30 * time.Second,
-			SnippetSize: 300,
-			Rules:       stores.Rules,
-			Retriever:   httpRetriever,
-			CFRetriever: cfRetriever,
-			CFRouteAll:  opts.CFRouteAll,
+			TimeOut:              30 * time.Second,
+			SnippetSize:          300,
+			Rules:                stores.Rules,
+			Retriever:            httpRetriever,
+			CFRetriever:          cfRetriever,
+			CFRouteAll:           opts.CFRouteAll,
+			BlockPrivateNetworks: blockPrivate,
 		},
 		Token:       opts.Token,
 		Credentials: opts.Credentials,