Skip to content

Commit cea78c5

Browse files
crawler: enforce Crawl-delay, back off on 429/503, surface politeness counters
Two field incidents drove this: a temporary IP ban from news.ycombinator.com (robots asks Crawl-delay: 30; we fetched at 1/s regardless) and a 424-response 429 storm from huggingface.co (an 823-item feed crawled with no backoff). - Crawl-delay was parsed but effectively unenforced: the delay was slept per-worker AFTER the host-gate slot was reserved, so concurrent workers still received slots at per_host_delay_ms spacing and the per-host request rate ignored robots entirely. robots.txt is now consulted before the gate and the delay widens the slot reservation itself (hostGate.WaitFor). Fractional Crawl-delay values parse, and a new crawler.max_crawl_delay_ms (default 2 min) clamps hostile/misconfigured values. - 429 (and 503 with Retry-After) no longer error the frontier entry: the host's next slot is deferred by Retry-After (or an exponential fallback, capped at 5 min) and the URL is requeued up to 3 times before failing. Rate-limited responses stay out of hostStats so the success-ratio blacklist can't be poisoned by a healthy-but-busy server. - Claim-time allowlist drops were invisible (imports report success while every item is discarded): now counted, logged once per host, and exposed together with deferral counts via /stats (crawl_dropped_disallowed, crawl_rate_limited_deferrals) and /metrics. - /admin/crawl-enqueue accepts an optional "lane" field (empty keeps the historical discovered default) so bulk feeds don't compete with discovery. - Tests: gate WaitFor/Defer timing under concurrency, fractional/negative Crawl-delay parsing, first fetchRSS coverage (RSS2/Atom/failure paths), 429 requeue-recover and give-up flows, enqueue lane routing.
1 parent 8be5707 commit cea78c5

12 files changed

Lines changed: 559 additions & 25 deletions

File tree

cmd/cosift/enqueue_lane_test.go

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
package main
2+
3+
import (
4+
"net/http"
5+
"net/http/httptest"
6+
"strings"
7+
"testing"
8+
)
9+
10+
// The lane field must be optional and backward compatible: an omitted/empty
11+
// lane keeps the historical default (Seed → discovered), never the
12+
// parseLaneName("") default (submitted).
13+
func TestHandleCrawlEnqueueLane(t *testing.T) {
14+
var seedCalls, laneCalls int
15+
var gotLane byte
16+
s := &pebbleHTTP{
17+
crawlSeed: func(string) error { seedCalls++; return nil },
18+
crawlSeedLane: func(_ string, lane byte) error { laneCalls++; gotLane = lane; return nil },
19+
}
20+
21+
post := func(body string) int {
22+
req := httptest.NewRequest(http.MethodPost, "/admin/crawl-enqueue", strings.NewReader(body))
23+
rec := httptest.NewRecorder()
24+
s.handleCrawlEnqueue(rec, req)
25+
return rec.Code
26+
}
27+
28+
if code := post(`{"url":"https://example.com/a"}`); code != http.StatusOK {
29+
t.Fatalf("no lane: code %d", code)
30+
}
31+
if seedCalls != 1 || laneCalls != 0 {
32+
t.Errorf("empty lane must use crawlSeed: seed=%d lane=%d", seedCalls, laneCalls)
33+
}
34+
35+
if code := post(`{"url":"https://example.com/b","lane":"bulk"}`); code != http.StatusOK {
36+
t.Fatalf("bulk lane: code %d", code)
37+
}
38+
if laneCalls != 1 || gotLane != 3 {
39+
t.Errorf("lane=bulk: laneCalls=%d gotLane=%d, want 1/3", laneCalls, gotLane)
40+
}
41+
42+
if code := post(`{"url":"https://example.com/c","lane":"refresh"}`); code != http.StatusOK {
43+
t.Fatalf("refresh lane: code %d", code)
44+
}
45+
if gotLane != 1 {
46+
t.Errorf("lane=refresh: gotLane=%d, want 1", gotLane)
47+
}
48+
}

cmd/cosift/serve_crawl.go

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,11 @@ import (
2121
// PeerAuthToken Bearer header.
2222
type crawlEnqueueReq struct {
2323
URL string `json:"url"`
24+
// Lane optionally targets a frontier lane ("submitted", "refresh",
25+
// "discovered", "bulk"). Empty keeps the historical default
26+
// (discovered) — note parseLaneName("") would mean submitted, so the
27+
// empty case must not be routed through it.
28+
Lane string `json:"lane,omitempty"`
2429
}
2530

2631
func (s *pebbleHTTP) handleCrawlEnqueue(w http.ResponseWriter, r *http.Request) {
@@ -42,7 +47,13 @@ func (s *pebbleHTTP) handleCrawlEnqueue(w http.ResponseWriter, r *http.Request)
4247
writeProblem(w, http.StatusBadRequest, "expected {\"url\": \"...\"}")
4348
return
4449
}
45-
if err := s.crawlSeed(req.URL); err != nil {
50+
var err error
51+
if req.Lane != "" && s.crawlSeedLane != nil {
52+
err = s.crawlSeedLane(req.URL, parseLaneName(req.Lane))
53+
} else {
54+
err = s.crawlSeed(req.URL)
55+
}
56+
if err != nil {
4657
writeProblem(w, http.StatusInternalServerError, err.Error())
4758
return
4859
}

cmd/cosift/serve_setup.go

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -792,6 +792,10 @@ func (s *pebbleHTTP) startInProcessCrawl(ctx context.Context, ps *store.PebbleSt
792792
}
793793
// Expose Seed so /admin/crawl-enqueue can hand off forwarded URLs.
794794
s.crawlSeed = c.Seed
795+
// expose SeedLane so /admin/crawl-enqueue can honor an explicit lane
796+
// (harvester bulk feeds shouldn't compete with discovery).
797+
s.crawlSeedLane = c.SeedLane
798+
s.crawlPoliteness = c.PolitenessStats
795799
// expose SeedSitemap so /admin/sitemap-import can push
796800
// sitemap-discovered URLs into the live frontier.
797801
s.crawlSeedSitemap = c.SeedSitemap
@@ -1027,6 +1031,11 @@ type pebbleHTTP struct {
10271031
// allowlist (used by /admin/allow-domain for organic HN/Reddit growth).
10281032
crawlAllowDomain func(domain string) error
10291033
crawlRecrawl func(ctx context.Context, url string) error
1034+
// crawlSeedLane backs the optional "lane" field on /admin/crawl-enqueue.
1035+
crawlSeedLane func(url string, lane byte) error
1036+
// crawlPoliteness feeds the allowlist-drop / rate-limit-deferral
1037+
// counters into /stats and /metrics.
1038+
crawlPoliteness func() (droppedDisallowed, rateDeferrals int64)
10301039

10311040
// doc count at startup so /stats can report crawl rate
10321041
// without persistent counter tables. docs_added = current - startup,

cmd/cosift/serve_stats.go

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -455,6 +455,11 @@ func (s *pebbleHTTP) buildStatsBody(ctx context.Context) ([]byte, error) {
455455
out["crawl_active"] = true
456456
out["docs_added_since_start"] = added
457457
out["docs_per_minute"] = rate
458+
if s.crawlPoliteness != nil {
459+
dropped, deferred := s.crawlPoliteness()
460+
out["crawl_dropped_disallowed"] = dropped
461+
out["crawl_rate_limited_deferrals"] = deferred
462+
}
458463
}
459464
return json.Marshal(out)
460465
}
@@ -515,6 +520,15 @@ func (s *pebbleHTTP) handleMetrics(w http.ResponseWriter, r *http.Request) {
515520
fmt.Fprintf(w, "# HELP cosift_crawl_docs_per_minute Recent crawl rate (docs added per minute, averaged since process start).\n")
516521
fmt.Fprintf(w, "# TYPE cosift_crawl_docs_per_minute gauge\n")
517522
fmt.Fprintf(w, "cosift_crawl_docs_per_minute %.2f\n", rate)
523+
if s.crawlPoliteness != nil {
524+
dropped, deferred := s.crawlPoliteness()
525+
fmt.Fprintf(w, "# HELP cosift_crawl_dropped_disallowed_total Frontier claims skipped because the domain is not allowlisted.\n")
526+
fmt.Fprintf(w, "# TYPE cosift_crawl_dropped_disallowed_total counter\n")
527+
fmt.Fprintf(w, "cosift_crawl_dropped_disallowed_total %d\n", dropped)
528+
fmt.Fprintf(w, "# HELP cosift_crawl_rate_limited_deferrals_total Host backoffs triggered by 429/503 rate-limit responses.\n")
529+
fmt.Fprintf(w, "# TYPE cosift_crawl_rate_limited_deferrals_total counter\n")
530+
fmt.Fprintf(w, "cosift_crawl_rate_limited_deferrals_total %d\n", deferred)
531+
}
518532
}
519533
// HyDE cache effectiveness. Hits/misses both monotonic so
520534
// Prometheus rate() over these gives cache pressure under load.

internal/config/config.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,12 @@ type Crawler struct {
118118
// "per_host_overrides": {"slow-server.example.com": 5000, "fast-cdn.com": 50}
119119
PerHostOverrides map[string]int `json:"per_host_overrides,omitempty"`
120120

121+
// MaxCrawlDelayMs caps the robots.txt Crawl-delay we honor. Without a
122+
// ceiling, a hostile or misconfigured robots.txt ("Crawl-delay: 86400")
123+
// would wedge its host for the whole 24h robots-cache TTL. 0 = default
124+
// (120000 = 2 min).
125+
MaxCrawlDelayMs int `json:"max_crawl_delay_ms,omitempty"`
126+
121127
// MaxBodyBytes caps the response size. Bigger pages get truncated.
122128
MaxBodyBytes int64 `json:"max_body_bytes"`
123129

internal/crawler/crawler.go

Lines changed: 168 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,22 @@ type Crawler struct {
104104
// re-enqueuing the same dead URLs the sweeper just purged.
105105
autoBlocked sync.Map // host (string) → struct{}
106106

107+
// Rate-limit bookkeeping. consec429 counts consecutive 429/503
108+
// responses per host (reset on any success) and drives the exponential
109+
// backoff when the server sends no Retry-After. rateRequeues bounds the
110+
// retry loop per URL — a host-level bound would fail every URL claimed
111+
// during one sustained throttle burst, even though the host recovers;
112+
// entries are removed on success or give-up, so the map only holds
113+
// currently-throttled URLs. rateDeferrals and droppedDisallowed feed
114+
// /stats; without the drop counter, allowlist fast-skips are invisible
115+
// to operators (imports appear to succeed while every item is
116+
// discarded at claim time).
117+
consec429 sync.Map // host (string) → *atomic.Int32
118+
rateRequeues sync.Map // url (string) → *atomic.Int32
119+
rateDeferrals atomic.Int64
120+
droppedDisallowed atomic.Int64
121+
dropLoggedHosts sync.Map // host (string) → struct{}, log once per host
122+
107123
// Dynamic allowlist: runtime-promoted domains that allowedDomain accepts
108124
// IN ADDITION to the static cfg.IncludeDomains. Lets curated sources
109125
// (HN/Reddit harvesters) grow the crawlable set organically — a domain
@@ -532,6 +548,13 @@ func (c *Crawler) SeedLane(rawURL string, lane byte) error {
532548
return c.store.PushFrontierLane(context.Background(), canon, 0, lane, 1.0)
533549
}
534550

551+
// PolitenessStats reports claim-time allowlist drops and rate-limit host
552+
// deferrals since start. Surfaced via /stats and /metrics — allowlist drops
553+
// are otherwise invisible (imports report success while items vanish).
554+
func (c *Crawler) PolitenessStats() (droppedDisallowed, rateDeferrals int64) {
555+
return c.droppedDisallowed.Load(), c.rateDeferrals.Load()
556+
}
557+
535558
// Recrawl re-enqueues a URL even if it was previously crawled. Status flips
536559
// to 'queued', attempts resets. Combined with the content-hash dedup in
537560
// processClaimed, an unchanged page costs one HTTP request and zero embedding
@@ -812,8 +835,48 @@ func (c *Crawler) worker(ctx context.Context, wg *sync.WaitGroup, gate *hostGate
812835
continue
813836
}
814837
err = c.processClaimed(ctx, item, gate)
815-
if u, perr := url.Parse(item.URL); perr == nil && u.Host != "" {
816-
c.recordHostResult(u.Host, err == nil)
838+
host := ""
839+
if u, perr := url.Parse(item.URL); perr == nil {
840+
host = u.Host
841+
}
842+
var rle *rateLimitedError
843+
if errors.As(err, &rle) && host != "" {
844+
// A throttling host is not a failing host. Back the whole host
845+
// off and requeue the URL (bounded per URL) instead of erroring
846+
// it — errored entries are terminal, and it stays out of
847+
// hostStats so the success-ratio blacklist can't be poisoned by
848+
// a healthy-but-busy server.
849+
d := rle.retryAfter
850+
if d == 0 {
851+
d = c.backoffDelay(c.bumpConsec429(host))
852+
} else {
853+
c.bumpConsec429(host)
854+
}
855+
if d > maxRateLimitDefer {
856+
d = maxRateLimitDefer
857+
}
858+
gate.Defer(host, d)
859+
c.rateDeferrals.Add(1)
860+
rv, _ := c.rateRequeues.LoadOrStore(item.URL, &atomic.Int32{})
861+
if n := rv.(*atomic.Int32).Add(1); n <= maxRateLimitRequeues {
862+
log.Printf("crawl %s: %v — host deferred %s, requeued (%d/%d)",
863+
item.URL, err, d, n, maxRateLimitRequeues)
864+
_ = c.store.RecrawlURL(ctx, item.URL)
865+
} else {
866+
log.Printf("crawl %s: %v — giving up after %d requeues", item.URL, err, maxRateLimitRequeues)
867+
c.rateRequeues.Delete(item.URL)
868+
_ = c.store.FailFrontier(ctx, item.URL, err.Error())
869+
}
870+
continue
871+
}
872+
// Any non-throttled outcome (success or terminal error) ends the
873+
// URL's requeue budget tracking.
874+
c.rateRequeues.Delete(item.URL)
875+
if host != "" {
876+
c.recordHostResult(host, err == nil)
877+
if err == nil {
878+
c.resetConsec429(host)
879+
}
817880
}
818881
if err != nil {
819882
log.Printf("crawl %s: %v", item.URL, err)
@@ -990,6 +1053,40 @@ func (c *Crawler) recordHostResult(host string, success bool) {
9901053
}
9911054
}
9921055

1056+
const (
1057+
// maxRateLimitRequeues bounds how many times one URL rides the
1058+
// 429-requeue loop before it errors out like any other failure.
1059+
maxRateLimitRequeues = 3
1060+
// maxRateLimitDefer caps a host backoff regardless of what
1061+
// Retry-After asks for — a worker slot blocked behind a deferred
1062+
// host must not stall for hours.
1063+
maxRateLimitDefer = 5 * time.Minute
1064+
)
1065+
1066+
func (c *Crawler) bumpConsec429(host string) int32 {
1067+
v, _ := c.consec429.LoadOrStore(host, &atomic.Int32{})
1068+
return v.(*atomic.Int32).Add(1)
1069+
}
1070+
1071+
func (c *Crawler) resetConsec429(host string) {
1072+
if v, ok := c.consec429.Load(host); ok {
1073+
v.(*atomic.Int32).Store(0)
1074+
}
1075+
}
1076+
1077+
// backoffDelay is the no-Retry-After fallback: per-host delay doubled per
1078+
// consecutive 429, from a 1s floor (tests run with PerHostDelayMs=0).
1079+
func (c *Crawler) backoffDelay(n int32) time.Duration {
1080+
base := time.Duration(c.cfg.PerHostDelayMs) * time.Millisecond
1081+
if base < time.Second {
1082+
base = time.Second
1083+
}
1084+
if n > 8 {
1085+
n = 8
1086+
}
1087+
return base << uint(n)
1088+
}
1089+
9931090
// terminator cancels runCtx once the frontier has stayed empty (no queued, no
9941091
// in-flight) across several polls. Three empties at 500ms = ~1.5s of quiet.
9951092
func (c *Crawler) terminator(ctx context.Context, cancel context.CancelFunc) {
@@ -1053,6 +1150,12 @@ func (c *Crawler) processClaimed(ctx context.Context, item store.FrontierItem, g
10531150
// the cursor to reach quality hosts. Returning nil signals success
10541151
// to the worker so the frontier entry transitions queued → done.
10551152
if !c.allowedDomain(item.URL) {
1153+
c.droppedDisallowed.Add(1)
1154+
if u != nil {
1155+
if _, logged := c.dropLoggedHosts.LoadOrStore(u.Host, struct{}{}); !logged {
1156+
log.Printf("crawl: dropping %s (domain not allowlisted; further drops for this host not logged)", item.URL)
1157+
}
1158+
}
10561159
return nil
10571160
}
10581161
// After we've tried a host N times
@@ -1062,33 +1165,35 @@ func (c *Crawler) processClaimed(ctx context.Context, item store.FrontierItem, g
10621165
if u != nil && c.isHostBlacklisted(u.Host) {
10631166
return nil
10641167
}
1065-
if u != nil {
1066-
// first time we see a host, fire-and-forget a sitemap
1067-
// discovery. Compounds: each new host typically brings hundreds-
1068-
// to-thousands of URLs in one batch.
1069-
c.maybeAutoSitemap(ctx, u)
1070-
if err := gate.Wait(ctx, u.Host); err != nil {
1071-
return err
1072-
}
1073-
}
1168+
// robots.txt before the gate: the Crawl-delay verdict must be known when
1169+
// the slot is reserved so it can widen the reservation itself. Sleeping
1170+
// after Wait (the previous approach) only shifted each worker's fetch by
1171+
// a constant while slots kept being granted at the configured interval —
1172+
// the effective per-host rate ignored Crawl-delay entirely. Cache hits
1173+
// are a map read; a miss fetches robots.txt once per host per TTL.
1174+
var robotsDelay time.Duration
10741175
if c.robots != nil {
1075-
allowed, robotsDelay, err := c.robots.Allowed(ctx, item.URL)
1176+
allowed, d, err := c.robots.Allowed(ctx, item.URL)
10761177
if err != nil {
10771178
return fmt.Errorf("robots: %w", err)
10781179
}
10791180
if !allowed {
10801181
return errors.New("blocked by robots.txt")
10811182
}
1082-
// Honor Crawl-delay if it exceeds our per-host gate's interval.
1083-
// use the effective per-host delay (gate's override or
1084-
// default), not the global default — operators who set a longer
1085-
// override for a host should have that override compared against
1086-
// robots.txt's Crawl-delay, not the global setting.
1087-
if robotsDelay > 0 && u != nil {
1088-
gateDelay := gate.delayFor(u.Host)
1089-
if robotsDelay > gateDelay {
1090-
sleepCtx(ctx, robotsDelay-gateDelay)
1091-
}
1183+
// Clamp: a misconfigured "Crawl-delay: 86400" must not wedge the
1184+
// host for the whole robots-cache TTL.
1185+
if maxDelay := c.maxCrawlDelay(); d > maxDelay {
1186+
d = maxDelay
1187+
}
1188+
robotsDelay = d
1189+
}
1190+
if u != nil {
1191+
// first time we see a host, fire-and-forget a sitemap
1192+
// discovery. Compounds: each new host typically brings hundreds-
1193+
// to-thousands of URLs in one batch.
1194+
c.maybeAutoSitemap(ctx, u)
1195+
if err := gate.WaitFor(ctx, u.Host, robotsDelay); err != nil {
1196+
return err
10921197
}
10931198
}
10941199

@@ -1566,6 +1671,42 @@ func (c *Crawler) allowedDomain(rawURL string) bool {
15661671
return false
15671672
}
15681673

1674+
func (c *Crawler) maxCrawlDelay() time.Duration {
1675+
if c.cfg.MaxCrawlDelayMs > 0 {
1676+
return time.Duration(c.cfg.MaxCrawlDelayMs) * time.Millisecond
1677+
}
1678+
return 2 * time.Minute
1679+
}
1680+
1681+
// rateLimitedError marks a 429 (or 503 with Retry-After) so the worker can
1682+
// back the host off and requeue instead of erroring the frontier entry —
1683+
// errored entries are terminal, and treating throttles as failures is what
1684+
// turns them into IP bans. retryAfter is 0 when the server sent no header.
1685+
type rateLimitedError struct {
1686+
code int
1687+
retryAfter time.Duration
1688+
}
1689+
1690+
func (e *rateLimitedError) Error() string { return fmt.Sprintf("http %d (rate limited)", e.code) }
1691+
1692+
// parseRetryAfter handles both forms the header allows: delta-seconds and
1693+
// HTTP-date. Returns 0 for anything unparseable or in the past.
1694+
func parseRetryAfter(v string) time.Duration {
1695+
v = strings.TrimSpace(v)
1696+
if v == "" {
1697+
return 0
1698+
}
1699+
if secs, err := strconv.Atoi(v); err == nil && secs > 0 {
1700+
return time.Duration(secs) * time.Second
1701+
}
1702+
if t, err := http.ParseTime(v); err == nil {
1703+
if d := time.Until(t); d > 0 {
1704+
return d
1705+
}
1706+
}
1707+
return 0
1708+
}
1709+
15691710
// fetchResult bundles the fetch outcome so processClaimed can distinguish
15701711
// "fresh body to parse" from "server said 304, nothing changed".
15711712
type fetchResult struct {
@@ -1612,6 +1753,11 @@ func (c *Crawler) fetch(ctx context.Context, u string, prior *store.Document) (*
16121753
etag: resp.Header.Get("ETag"), lastModified: resp.Header.Get("Last-Modified")}, nil
16131754
}
16141755
if resp.StatusCode >= 400 {
1756+
retryAfter := parseRetryAfter(resp.Header.Get("Retry-After"))
1757+
if resp.StatusCode == http.StatusTooManyRequests ||
1758+
(resp.StatusCode == http.StatusServiceUnavailable && retryAfter > 0) {
1759+
return nil, &rateLimitedError{code: resp.StatusCode, retryAfter: retryAfter}
1760+
}
16151761
return nil, fmt.Errorf("http %d", resp.StatusCode)
16161762
}
16171763
ct := resp.Header.Get("Content-Type")

0 commit comments

Comments
 (0)