@@ -104,6 +104,22 @@ type Crawler struct {
104104 // re-enqueuing the same dead URLs the sweeper just purged.
105105 autoBlocked sync.Map // host (string) → struct{}
106106
107+ // Rate-limit bookkeeping. consec429 counts consecutive 429/503
108+ // responses per host (reset on any success) and drives the exponential
109+ // backoff when the server sends no Retry-After. rateRequeues bounds the
110+ // retry loop per URL — a host-level bound would fail every URL claimed
111+ // during one sustained throttle burst, even though the host recovers;
112+ // entries are removed on success or give-up, so the map only holds
113+ // currently-throttled URLs. rateDeferrals and droppedDisallowed feed
114+ // /stats; without the drop counter, allowlist fast-skips are invisible
115+ // to operators (imports appear to succeed while every item is
116+ // discarded at claim time).
117+ consec429 sync.Map // host (string) → *atomic.Int32
118+ rateRequeues sync.Map // url (string) → *atomic.Int32
119+ rateDeferrals atomic.Int64
120+ droppedDisallowed atomic.Int64
121+ dropLoggedHosts sync.Map // host (string) → struct{}, log once per host
122+
107123 // Dynamic allowlist: runtime-promoted domains that allowedDomain accepts
108124 // IN ADDITION to the static cfg.IncludeDomains. Lets curated sources
109125 // (HN/Reddit harvesters) grow the crawlable set organically — a domain
@@ -532,6 +548,13 @@ func (c *Crawler) SeedLane(rawURL string, lane byte) error {
532548 return c .store .PushFrontierLane (context .Background (), canon , 0 , lane , 1.0 )
533549}
534550
551+ // PolitenessStats reports claim-time allowlist drops and rate-limit host
552+ // deferrals since start. Surfaced via /stats and /metrics — allowlist drops
553+ // are otherwise invisible (imports report success while items vanish).
554+ func (c * Crawler ) PolitenessStats () (droppedDisallowed , rateDeferrals int64 ) {
555+ return c .droppedDisallowed .Load (), c .rateDeferrals .Load ()
556+ }
557+
535558// Recrawl re-enqueues a URL even if it was previously crawled. Status flips
536559// to 'queued', attempts resets. Combined with the content-hash dedup in
537560// processClaimed, an unchanged page costs one HTTP request and zero embedding
@@ -812,8 +835,48 @@ func (c *Crawler) worker(ctx context.Context, wg *sync.WaitGroup, gate *hostGate
812835 continue
813836 }
814837 err = c .processClaimed (ctx , item , gate )
815- if u , perr := url .Parse (item .URL ); perr == nil && u .Host != "" {
816- c .recordHostResult (u .Host , err == nil )
838+ host := ""
839+ if u , perr := url .Parse (item .URL ); perr == nil {
840+ host = u .Host
841+ }
842+ var rle * rateLimitedError
843+ if errors .As (err , & rle ) && host != "" {
844+ // A throttling host is not a failing host. Back the whole host
845+ // off and requeue the URL (bounded per URL) instead of erroring
846+ // it — errored entries are terminal, and it stays out of
847+ // hostStats so the success-ratio blacklist can't be poisoned by
848+ // a healthy-but-busy server.
849+ d := rle .retryAfter
850+ if d == 0 {
851+ d = c .backoffDelay (c .bumpConsec429 (host ))
852+ } else {
853+ c .bumpConsec429 (host )
854+ }
855+ if d > maxRateLimitDefer {
856+ d = maxRateLimitDefer
857+ }
858+ gate .Defer (host , d )
859+ c .rateDeferrals .Add (1 )
860+ rv , _ := c .rateRequeues .LoadOrStore (item .URL , & atomic.Int32 {})
861+ if n := rv .(* atomic.Int32 ).Add (1 ); n <= maxRateLimitRequeues {
862+ log .Printf ("crawl %s: %v — host deferred %s, requeued (%d/%d)" ,
863+ item .URL , err , d , n , maxRateLimitRequeues )
864+ _ = c .store .RecrawlURL (ctx , item .URL )
865+ } else {
866+ log .Printf ("crawl %s: %v — giving up after %d requeues" , item .URL , err , maxRateLimitRequeues )
867+ c .rateRequeues .Delete (item .URL )
868+ _ = c .store .FailFrontier (ctx , item .URL , err .Error ())
869+ }
870+ continue
871+ }
872+ // Any non-throttled outcome (success or terminal error) ends the
873+ // URL's requeue budget tracking.
874+ c .rateRequeues .Delete (item .URL )
875+ if host != "" {
876+ c .recordHostResult (host , err == nil )
877+ if err == nil {
878+ c .resetConsec429 (host )
879+ }
817880 }
818881 if err != nil {
819882 log .Printf ("crawl %s: %v" , item .URL , err )
@@ -990,6 +1053,40 @@ func (c *Crawler) recordHostResult(host string, success bool) {
9901053 }
9911054}
9921055
1056+ const (
1057+ // maxRateLimitRequeues bounds how many times one URL rides the
1058+ // 429-requeue loop before it errors out like any other failure.
1059+ maxRateLimitRequeues = 3
1060+ // maxRateLimitDefer caps a host backoff regardless of what
1061+ // Retry-After asks for — a worker slot blocked behind a deferred
1062+ // host must not stall for hours.
1063+ maxRateLimitDefer = 5 * time .Minute
1064+ )
1065+
1066+ func (c * Crawler ) bumpConsec429 (host string ) int32 {
1067+ v , _ := c .consec429 .LoadOrStore (host , & atomic.Int32 {})
1068+ return v .(* atomic.Int32 ).Add (1 )
1069+ }
1070+
1071+ func (c * Crawler ) resetConsec429 (host string ) {
1072+ if v , ok := c .consec429 .Load (host ); ok {
1073+ v .(* atomic.Int32 ).Store (0 )
1074+ }
1075+ }
1076+
1077+ // backoffDelay is the no-Retry-After fallback: per-host delay doubled per
1078+ // consecutive 429, from a 1s floor (tests run with PerHostDelayMs=0).
1079+ func (c * Crawler ) backoffDelay (n int32 ) time.Duration {
1080+ base := time .Duration (c .cfg .PerHostDelayMs ) * time .Millisecond
1081+ if base < time .Second {
1082+ base = time .Second
1083+ }
1084+ if n > 8 {
1085+ n = 8
1086+ }
1087+ return base << uint (n )
1088+ }
1089+
9931090// terminator cancels runCtx once the frontier has stayed empty (no queued, no
9941091// in-flight) across several polls. Three empties at 500ms = ~1.5s of quiet.
9951092func (c * Crawler ) terminator (ctx context.Context , cancel context.CancelFunc ) {
@@ -1053,6 +1150,12 @@ func (c *Crawler) processClaimed(ctx context.Context, item store.FrontierItem, g
10531150 // the cursor to reach quality hosts. Returning nil signals success
10541151 // to the worker so the frontier entry transitions queued → done.
10551152 if ! c .allowedDomain (item .URL ) {
1153+ c .droppedDisallowed .Add (1 )
1154+ if u != nil {
1155+ if _ , logged := c .dropLoggedHosts .LoadOrStore (u .Host , struct {}{}); ! logged {
1156+ log .Printf ("crawl: dropping %s (domain not allowlisted; further drops for this host not logged)" , item .URL )
1157+ }
1158+ }
10561159 return nil
10571160 }
10581161 // After we've tried a host N times
@@ -1062,33 +1165,35 @@ func (c *Crawler) processClaimed(ctx context.Context, item store.FrontierItem, g
10621165 if u != nil && c .isHostBlacklisted (u .Host ) {
10631166 return nil
10641167 }
1065- if u != nil {
1066- // first time we see a host, fire-and-forget a sitemap
1067- // discovery. Compounds: each new host typically brings hundreds-
1068- // to-thousands of URLs in one batch.
1069- c .maybeAutoSitemap (ctx , u )
1070- if err := gate .Wait (ctx , u .Host ); err != nil {
1071- return err
1072- }
1073- }
1168+ // robots.txt before the gate: the Crawl-delay verdict must be known when
1169+ // the slot is reserved so it can widen the reservation itself. Sleeping
1170+ // after Wait (the previous approach) only shifted each worker's fetch by
1171+ // a constant while slots kept being granted at the configured interval —
1172+ // the effective per-host rate ignored Crawl-delay entirely. Cache hits
1173+ // are a map read; a miss fetches robots.txt once per host per TTL.
1174+ var robotsDelay time.Duration
10741175 if c .robots != nil {
1075- allowed , robotsDelay , err := c .robots .Allowed (ctx , item .URL )
1176+ allowed , d , err := c .robots .Allowed (ctx , item .URL )
10761177 if err != nil {
10771178 return fmt .Errorf ("robots: %w" , err )
10781179 }
10791180 if ! allowed {
10801181 return errors .New ("blocked by robots.txt" )
10811182 }
1082- // Honor Crawl-delay if it exceeds our per-host gate's interval.
1083- // use the effective per-host delay (gate's override or
1084- // default), not the global default — operators who set a longer
1085- // override for a host should have that override compared against
1086- // robots.txt's Crawl-delay, not the global setting.
1087- if robotsDelay > 0 && u != nil {
1088- gateDelay := gate .delayFor (u .Host )
1089- if robotsDelay > gateDelay {
1090- sleepCtx (ctx , robotsDelay - gateDelay )
1091- }
1183+ // Clamp: a misconfigured "Crawl-delay: 86400" must not wedge the
1184+ // host for the whole robots-cache TTL.
1185+ if maxDelay := c .maxCrawlDelay (); d > maxDelay {
1186+ d = maxDelay
1187+ }
1188+ robotsDelay = d
1189+ }
1190+ if u != nil {
1191+ // first time we see a host, fire-and-forget a sitemap
1192+ // discovery. Compounds: each new host typically brings hundreds-
1193+ // to-thousands of URLs in one batch.
1194+ c .maybeAutoSitemap (ctx , u )
1195+ if err := gate .WaitFor (ctx , u .Host , robotsDelay ); err != nil {
1196+ return err
10921197 }
10931198 }
10941199
@@ -1566,6 +1671,42 @@ func (c *Crawler) allowedDomain(rawURL string) bool {
15661671 return false
15671672}
15681673
1674+ func (c * Crawler ) maxCrawlDelay () time.Duration {
1675+ if c .cfg .MaxCrawlDelayMs > 0 {
1676+ return time .Duration (c .cfg .MaxCrawlDelayMs ) * time .Millisecond
1677+ }
1678+ return 2 * time .Minute
1679+ }
1680+
1681+ // rateLimitedError marks a 429 (or 503 with Retry-After) so the worker can
1682+ // back the host off and requeue instead of erroring the frontier entry —
1683+ // errored entries are terminal, and treating throttles as failures is what
1684+ // turns them into IP bans. retryAfter is 0 when the server sent no header.
1685+ type rateLimitedError struct {
1686+ code int
1687+ retryAfter time.Duration
1688+ }
1689+
1690+ func (e * rateLimitedError ) Error () string { return fmt .Sprintf ("http %d (rate limited)" , e .code ) }
1691+
1692+ // parseRetryAfter handles both forms the header allows: delta-seconds and
1693+ // HTTP-date. Returns 0 for anything unparseable or in the past.
1694+ func parseRetryAfter (v string ) time.Duration {
1695+ v = strings .TrimSpace (v )
1696+ if v == "" {
1697+ return 0
1698+ }
1699+ if secs , err := strconv .Atoi (v ); err == nil && secs > 0 {
1700+ return time .Duration (secs ) * time .Second
1701+ }
1702+ if t , err := http .ParseTime (v ); err == nil {
1703+ if d := time .Until (t ); d > 0 {
1704+ return d
1705+ }
1706+ }
1707+ return 0
1708+ }
1709+
15691710// fetchResult bundles the fetch outcome so processClaimed can distinguish
15701711// "fresh body to parse" from "server said 304, nothing changed".
15711712type fetchResult struct {
@@ -1612,6 +1753,11 @@ func (c *Crawler) fetch(ctx context.Context, u string, prior *store.Document) (*
16121753 etag : resp .Header .Get ("ETag" ), lastModified : resp .Header .Get ("Last-Modified" )}, nil
16131754 }
16141755 if resp .StatusCode >= 400 {
1756+ retryAfter := parseRetryAfter (resp .Header .Get ("Retry-After" ))
1757+ if resp .StatusCode == http .StatusTooManyRequests ||
1758+ (resp .StatusCode == http .StatusServiceUnavailable && retryAfter > 0 ) {
1759+ return nil , & rateLimitedError {code : resp .StatusCode , retryAfter : retryAfter }
1760+ }
16151761 return nil , fmt .Errorf ("http %d" , resp .StatusCode )
16161762 }
16171763 ct := resp .Header .Get ("Content-Type" )
0 commit comments