Stop declaring requests directly and fix the sources that used it

The scraper already brings requests[socks] in, so lncrawl declaring it too pinned a
version it does not own. The browser middleware now uses starlette's own mutable
headers, and the four sources that reached past the scraper go through it: chireads
and wtrlab work as before, fenrirscan keeps returning nothing rather than raising,
and totallytranslations no longer replaces its session with one that has no scraper
methods.

Base URLs are deduplicated in declared order with https first, instead of through a
set. base_url[0] is home_url and the scraper's origin, so which address that was
changed between runs of identical code, and three sources defaulted to plaintext http.

Author: dipu-bd

CHANGELOG.md

@@ -151,6 +151,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   because the default cannot handle their markup; the value was stored on the crawler
   and never reached the session that builds the soup.
 
+- **A source's primary address no longer changes between runs.** Where a source lists
+  several addresses, the one treated as its home was picked from an unordered set, so it
+  varied from one run to the next for the same unchanged code — and a source listing both
+  `http` and `https` for the same site could end up pinned to the plaintext one. Secure
+  addresses now come first, the source's own order is kept within that, and three sources
+  stop defaulting to `http`. This also makes the generated source index reproducible: it
+  was rewriting a hundred lines on every build with nothing behind it.
+
+- **`chireads` search results had no titles.** The site moved the title out of the
+  attribute the source read; it now reads the one the site actually emits.
+  `totallytranslations` replaced its own HTTP session on startup, which broke every
+  request it then made — that domain has since lapsed and was already flagged as
+  unavailable, so this only removes the broken code.
+
 - **Asking what is known about a site no longer records that it was asked.** Reading a
   site's diagnosis stored a profile for it, so browsing the sources list wrote one entry
   per site looked at — and with the store bounded, that could push out what a running

lncrawl/server/middleware/browser.py

@@ -5,9 +5,9 @@
 import pickle
 from urllib.parse import urljoin, urlparse
 
-from requests.structures import CaseInsensitiveDict
 from scraper import Scraper
 from scraper.exceptions import Blocked
+from starlette.datastructures import MutableHeaders
 from starlette.requests import Request
 from starlette.responses import RedirectResponse, Response
 
@@ -78,7 +78,7 @@ async def _proxy(self, scraper: Scraper, path: str, request: Request) -> Respons
 
         body = await request.body()
 
-        headers = CaseInsensitiveDict(request.headers)
+        headers = MutableHeaders(headers=dict(request.headers))
         if headers.get("host"):
             headers["host"] = apex_domain
         if headers.get("origin"):

lncrawl/services/jobs/service.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import Any, Callable, Iterable, List, Optional, TypeVar, Union
+from typing import Any, Callable, Dict, Iterable, List, Optional, TypeVar, Union
 
 from sqlalchemy.orm import aliased
 import sqlmodel as sq
@@ -1002,20 +1002,20 @@ def _fail(
             done=Job.done + pending,
             failed=Job.failed + pending,
         )
-        extra = self._get_extra(sess, job_id, extra)
         self._update(
             sess,
             job_id,
             error=reason,
             status=JobStatus.FAILED,
+            extra=self._get_extra(sess, job_id, extra),
         )
 
     def _get_extra(
         self,
         sess: Session,
         job_id: str,
         updates: Union[dict, Callable[[dict], None]],
-    ) -> None:
+    ) -> Dict[str, Any]:
         current = sess.scalar(sq.select(Job.extra).where(Job.id == job_id))
 
         extra = dict(current or {})

pyproject.toml

@@ -56,7 +56,6 @@ dependencies = [
   "exejs>=0.0.7",
   "readability-lxml>=0.8.0,<1.0.0",
   "regex>=2024.0.0",
-  "requests[socks]>=2.32.0",
   "sqlmodel>=0.0.22",
   "tqdm>=4.66.0,<5.0.0",
   "typer>=0.12.0",

sources/en/t/totallytranslations.py

@@ -1,8 +1,6 @@
 # -*- coding: utf-8 -*-
 import logging
 
-from requests.sessions import Session
-
 from lncrawl.core import Chapter, LegacyCrawler
 
 logger = logging.getLogger(__name__)
@@ -11,9 +9,6 @@
 class TotallyTranslations(LegacyCrawler):
     base_url = "https://totallytranslations.com/"
 
-    def initialize(self):
-        self.scraper = Session()
-
     def read_novel_info(self):
         logger.debug("Visiting %s", self.novel_url)
         soup = self.get_soup(self.novel_url)

sources/fr/chireads.py

@@ -2,8 +2,6 @@
 
 import logging
 
-import requests
-
 from lncrawl.core import Chapter, LegacyCrawler
 
 logger = logging.getLogger(__name__)
@@ -17,19 +15,17 @@ class Chireads(LegacyCrawler):
     def search_novel(self, query):
         query = query.lower().replace(" ", "+")
 
-        # NOTE: Using self.get_soup() here throw an error, I don't know why.
-        response = requests.get("https://chireads.com/search?x=0&y=0&name=" + query)
-        soup = self.make_soup(response)
+        soup = self.get_soup("https://chireads.com/search?x=0&y=0&name=" + query)
 
         result = []
         content = soup.find("div", {"id": "content"})
 
         for novel in content.find_all("li"):
-            content = novel.find("a")
+            link = novel.find("a")
             result.append(
                 {
-                    "title": content.get("title"),
-                    "url": self.absolute_url(content.get("href")),
+                    "title": link.get("aria-label"),
+                    "url": self.absolute_url(link.get("href")),
                 }
             )
 

sources/multi/wtrlab.py

@@ -6,7 +6,6 @@
 from typing import List, Union
 
 from cryptography.hazmat.primitives.ciphers.aead import AESGCM
-import requests
 
 from lncrawl.core import Chapter, LegacyCrawler, PageSoup, SearchResult
 
@@ -25,10 +24,10 @@ def _parse_next_data(self, soup: PageSoup):
         return json.loads(metadata_json.get_text(strip=True))
 
     def search_novel(self, query: str):
-        novels = requests.post(
+        novels = self.post_json(
             "https://www.wtr-lab.com/api/search",
             json={"text": query},
-        ).json()
+        )
         logger.info("Search results: %s", novels)
 
         results = []

sources/tr/fenrirscan.py

@@ -2,8 +2,6 @@
 import logging
 import re
 
-import requests
-
 from lncrawl.core import Chapter, LegacyCrawler, SearchResult
 
 logger = logging.getLogger(__name__)
@@ -16,16 +14,17 @@ class FenrirScans(LegacyCrawler):
     has_mtl = False
 
     def search_novel(self, query):
-        """
-        Uses the site's AJAX search endpoint to find novels.
-        """
-        # Prepare payload for AJAX search
+        # The site no longer answers this: its theme registers only next/prev/tepki, and
+        # `ts_ac_do_search` returns `400`. Its `?s=` page is no better — the grid it
+        # renders is byte-identical for every query, including one that matches nothing.
+        # Left in place because a restored endpoint would work again as written, and
+        # returning nothing is what this did before the request began raising.
         data = {
             "action": "ts_ac_do_search",
             "ts_ac_query": query,
         }
-        response = requests.post(self.search_url, data=data)
         try:
+            response = self.post_response(self.search_url, data=data)
             return [
                 SearchResult(
                     title=item["post_title"],
@@ -37,9 +36,6 @@ def search_novel(self, query):
             return []
 
     def read_novel_info(self):
-        """
-        Parses novel metadata and chapter list from the novel page.
-        """
         soup = self.get_soup(self.novel_url)
         # Metadata
         title = soup.find("h1")
@@ -66,8 +62,10 @@ def read_novel_info(self):
 
         # Chapters
         chapter_links = soup.find_all("a", href=re.compile(r"-bolum-\d+"))
+
         # Reverse so earliest chapters first
         chapter_links = list(reversed(chapter_links))
+
         for idx, a in enumerate(chapter_links, 1):
             chap_url = self.absolute_url(a["href"])
             chap_title = (a.find("span", {"class": "chapternum"}) or a).text