build.rs

use reqwest::blocking::Client;
use std::fs;
use std::path::{Path, PathBuf};

// Reuse HTML parsing logic from src/html_utils.rs
include!("src/html_utils.rs");

/// Generates test fixtures by fetching real data from package registries.
/// Runs during build if DOC2SKILL_REGEN_FIXTURES=1 or if fixtures don't exist.
fn main() {
    // Generate only when explicitly asked. Writing to the source tree from a
    // build script is forbidden by `cargo publish` (verification builds a copy
    // of the package and rejects source-tree mutations outside OUT_DIR), and
    // fetching from the network on every `cargo install`/CI build job is both
    // slow and fragile. CI's test job and devs set DOC2SKILL_REGEN_FIXTURES=1.
    let should_regen = std::env::var("DOC2SKILL_REGEN_FIXTURES")
        .map(|v| v == "1" || v.to_lowercase() == "true")
        .unwrap_or(false);

    if !should_regen {
        return;
    }

    let fixtures_dir = PathBuf::from("tests/fixtures_gen");
    println!("cargo:warning=Regenerating test fixtures from real APIs...");

    let client = Client::builder()
        .user_agent("doc2skill/0.1 (https://github.com/odonno/doc2skill)")
        .build()
        .expect("Failed to create HTTP client");

    // Rust packages
    generate_rust_fixtures(&client, &fixtures_dir, "clap", Some("4.6.1"));
    generate_rust_fixtures(&client, &fixtures_dir, "clap", None); // latest
    generate_rust_fixtures(&client, &fixtures_dir, "serde", None); // latest
    generate_rust_fixtures(&client, &fixtures_dir, "color-eyre", Some("0.6.5"));

    // TypeScript packages
    generate_typescript_fixtures(&client, &fixtures_dir, "react", Some("19.2.7"));
    generate_typescript_fixtures(&client, &fixtures_dir, "react", None); // latest

    // C# packages
    generate_csharp_fixtures(&client, &fixtures_dir, "Newtonsoft.Json", None);
    generate_csharp_fixtures(&client, &fixtures_dir, "System.Text.Json", Some("10.0.10"));

    // llms.txt sources (real external docs)
    generate_llmtxt_fixtures(
        &client,
        &fixtures_dir,
        "https://www.fastht.ml/docs/llms.txt",
    );
    generate_llmtxt_fixtures(&client, &fixtures_dir, "https://likec4.dev/llms-full.txt");

    println!("cargo:warning=Test fixtures generated successfully");
}

/// Fetches a real llms.txt (index) or llms-full.txt (self-contained) doc and writes
/// it under tests/fixtures_gen/llms-txt/ so the mock server can serve it.
///
/// For an index, each `## `-section link line is downloaded and the link URL is
/// rewritten to a relative path (the URL minus scheme+host) so the test stays
/// hermetic — url_to_slug of that relative path equals url_to_slug of the real
/// URL, so reference filenames match real runtime output.
fn generate_llmtxt_fixtures(client: &Client, fixtures_dir: &Path, url: &str) {
    println!("cargo:warning=Generating llms.txt fixtures for '{}'", url);

    let dest_dir = fixtures_dir.join("llms-txt");
    fs::create_dir_all(&dest_dir).unwrap();

    let resp = match client.get(url).send() {
        Ok(r) if r.status().is_success() => r,
        Ok(r) => {
            println!("cargo:warning=  skipped '{}' (status {})", url, r.status());
            return;
        }
        Err(e) => {
            println!("cargo:warning=  skipped '{}' ({})", url, e);
            return;
        }
    };
    let body = resp.text().expect("Failed to read llms.txt response");

    let basename = url.rsplit('/').next().unwrap_or("llms.txt");
    let is_index = basename == "llms.txt";

    if !is_index {
        // Self-contained full doc: save as-is.
        fs::write(dest_dir.join(basename), &body).expect("Failed to write llms-full fixture");
        return;
    }

    // Index form: parse link lines, fetch each linked page, rewrite URLs to relative.
    let mut rewritten = String::with_capacity(body.len());
    for line in body.lines() {
        let trimmed = line.trim_start();
        let entry = trimmed.strip_prefix("- [");
        let Some(rest) = entry else {
            rewritten.push_str(line);
            rewritten.push('\n');
            continue;
        };
        let Some(close) = rest.find(']') else {
            rewritten.push_str(line);
            rewritten.push('\n');
            continue;
        };
        let after = &rest[close + 1..];
        let Some(paren_start) = after.find('(') else {
            rewritten.push_str(line);
            rewritten.push('\n');
            continue;
        };
        let Some(paren_end) = after[paren_start + 1..].find(')') else {
            rewritten.push_str(line);
            rewritten.push('\n');
            continue;
        };
        let link_url = &after[paren_start + 1..paren_start + 1 + paren_end];

        if !link_url.starts_with("http://") && !link_url.starts_with("https://") {
            rewritten.push_str(line);
            rewritten.push('\n');
            continue;
        }

        let rel = to_relative_path(link_url);
        fetch_and_save(client, link_url, &dest_dir, rel);

        rewritten.push_str(&line.replace(link_url, rel));
        rewritten.push('\n');
    }
    fs::write(dest_dir.join("llms.txt"), rewritten).expect("Failed to write llms.txt fixture");
}

/// Strip scheme and host, returning the path (no leading slash).
fn to_relative_path(url: &str) -> &str {
    let after = url.split("://").nth(1).unwrap_or(url);
    after.find('/').map(|i| &after[i + 1..]).unwrap_or(after)
}

fn fetch_and_save(client: &Client, url: &str, dest_dir: &Path, rel: &str) {
    let Ok(resp) = client.get(url).send() else {
        println!(
            "cargo:warning=  skipped linked page '{}' (send failed)",
            url
        );
        return;
    };
    if !resp.status().is_success() {
        println!(
            "cargo:warning=  skipped linked page '{}' (status {})",
            url,
            resp.status()
        );
        return;
    }
    let body = resp.text().expect("Failed to read linked page");
    let path = dest_dir.join(rel);
    fs::create_dir_all(path.parent().unwrap()).unwrap();
    fs::write(&path, &body).expect("Failed to write linked page fixture");
}

fn generate_rust_fixtures(client: &Client, fixtures_dir: &Path, name: &str, version: Option<&str>) {
    println!(
        "cargo:warning=Generating Rust fixtures for '{}' {:?}",
        name, version
    );

    // Fetch crate metadata
    let url = format!("https://crates.io/api/v1/crates/{}", name);
    let resp = client
        .get(&url)
        .send()
        .expect("Failed to fetch crate metadata");
    let body = resp.text().expect("Failed to read response");
    let path = fixtures_dir.join(format!("crates.io/api/v1/crates/{}.json", name));
    fs::create_dir_all(path.parent().unwrap()).unwrap();
    fs::write(&path, &body).expect("Failed to write fixture");

    // Parse version from metadata or use provided version
    let metadata: serde_json::Value = serde_json::from_str(&body).unwrap();
    let version = version.map(|v| v.to_string()).unwrap_or_else(|| {
        metadata["crate"]["newest_version"]
            .as_str()
            .unwrap_or("latest")
            .to_string()
    });

    // Fetch crate owners
    let url = format!("https://crates.io/api/v1/crates/{}/owners", name);
    let resp = client
        .get(&url)
        .send()
        .expect("Failed to fetch crate owners");
    let body = resp.text().expect("Failed to read response");
    let path = fixtures_dir.join(format!("crates.io/api/v1/crates/{}/owners.json", name));
    fs::create_dir_all(path.parent().unwrap()).unwrap();
    fs::write(&path, &body).expect("Failed to write fixture");

    // Fetch docs index
    let module = name.replace('-', "_");
    let url = format!("https://docs.rs/{}/{}/{}/index.html", name, version, module);
    if let Ok(resp) = client.get(&url).send()
        && resp.status().is_success()
    {
        let body = resp.text().unwrap();
        let path = fixtures_dir.join(format!(
            "docs.rs/{}/{}/{}/index.html",
            name, version, module
        ));
        fs::create_dir_all(path.parent().unwrap()).unwrap();
        fs::write(&path, &body).expect("Failed to write fixture");

        let crate_base = format!("https://docs.rs/{}/{}/{}", name, version, module);

        // Extract reference URLs from index.html
        let mut reference_urls: Vec<(String, String)> = extract_html_references(&body, &crate_base);

        // Fetch all.html to get complete item list
        let all_url = format!("{}/all.html", crate_base);
        if let Ok(resp) = client.get(&all_url).send()
            && resp.status().is_success()
        {
            let body = resp.text().unwrap();
            let path =
                fixtures_dir.join(format!("docs.rs/{}/{}/{}/all.html", name, version, module));
            fs::create_dir_all(path.parent().unwrap()).unwrap();
            fs::write(&path, &body).expect("Failed to write fixture");

            let all_references = extract_html_references(&body, &crate_base);
            reference_urls.extend(all_references);
        }

        // Deduplicate by relative path
        let mut seen = HashSet::new();
        reference_urls.retain(|(_, rel)| seen.insert(rel.clone()));

        // Fetch each reference page
        for (full_url, rel_path) in reference_urls {
            if let Ok(resp) = client.get(&full_url).send()
                && resp.status().is_success()
            {
                let body = resp.text().unwrap();
                let path = fixtures_dir.join(format!(
                    "docs.rs/{}/{}/{}/{}",
                    name, version, module, rel_path
                ));
                fs::create_dir_all(path.parent().unwrap()).unwrap();
                fs::write(&path, &body).expect("Failed to write reference fixture");
            }
        }
    }
}

fn generate_typescript_fixtures(
    client: &Client,
    fixtures_dir: &Path,
    name: &str,
    version: Option<&str>,
) {
    println!(
        "cargo:warning=Generating TypeScript fixtures for '{}' {:?}",
        name, version
    );

    // Fetch package metadata (specific or latest version)
    let url = if let Some(v) = version {
        format!("https://registry.npmjs.org/{}/{}", name, v)
    } else {
        format!("https://registry.npmjs.org/{}/latest", name)
    };
    let resp = client
        .get(&url)
        .send()
        .expect("Failed to fetch package metadata");
    let body = resp.text().expect("Failed to read response");
    let path = if let Some(v) = version {
        fixtures_dir.join(format!("npm/{}/{}.json", name, v))
    } else {
        fixtures_dir.join(format!("npm/{}/latest.json", name))
    };
    fs::create_dir_all(path.parent().unwrap()).unwrap();
    fs::write(&path, &body).expect("Failed to write fixture");

    // Parse version from metadata
    let metadata: serde_json::Value = serde_json::from_str(&body).unwrap();
    let version = version
        .map(|v| v.to_string())
        .unwrap_or_else(|| metadata["version"].as_str().unwrap_or("latest").to_string());

    // Fetch README from jsdelivr
    let url = format!(
        "https://cdn.jsdelivr.net/npm/{}@{}/README.md",
        name, version
    );
    if let Ok(resp) = client.get(&url).send()
        && resp.status().is_success()
    {
        let body = resp.text().unwrap();
        let path = fixtures_dir.join(format!("jsdelivr/npm/{}@{}/README.md", name, version));
        fs::create_dir_all(path.parent().unwrap()).unwrap();
        fs::write(&path, &body).expect("Failed to write fixture");
    }

    // Fetch npmx docs
    let url = format!("https://npmx.dev/api/registry/docs/{}/v/{}", name, version);
    if let Ok(resp) = client.get(&url).send()
        && resp.status().is_success()
    {
        let body = resp.text().unwrap();
        let path = fixtures_dir.join(format!(
            "npmx/api/registry/docs/{}/v/{}.json",
            name, version
        ));
        fs::create_dir_all(path.parent().unwrap()).unwrap();
        fs::write(&path, &body).expect("Failed to write fixture");
    }
}

fn generate_csharp_fixtures(client: &Client, fixtures_dir: &Path, id: &str, version: Option<&str>) {
    println!(
        "cargo:warning=Generating C# fixtures for '{}' {:?}",
        id, version
    );

    let id_lower = id.to_lowercase();

    // Fetch versions
    let url = format!(
        "https://api.nuget.org/v3-flatcontainer/{}/index.json",
        id_lower
    );
    let resp = client.get(&url).send().expect("Failed to fetch versions");
    let body = resp.text().expect("Failed to read response");
    let path = fixtures_dir.join(format!("nuget/v3-flatcontainer/{}/index.json", id_lower));
    fs::create_dir_all(path.parent().unwrap()).unwrap();
    fs::write(&path, &body).expect("Failed to write fixture");

    // Parse version from parameter or use latest stable
    let versions: serde_json::Value = serde_json::from_str(&body).unwrap();
    let version = version.map(|v| v.to_string()).unwrap_or_else(|| {
        versions["versions"]
            .as_array()
            .and_then(|vs| {
                vs.iter()
                    .rev()
                    .find(|v| v.as_str().is_some_and(|s| !s.contains('-')))
            })
            .and_then(|v| v.as_str())
            .unwrap_or("latest")
            .to_string()
    });

    // Fetch registration leaf
    let url = format!(
        "https://api.nuget.org/v3/registration5-semver1/{}/{}.json",
        id_lower, version
    );
    let resp = client
        .get(&url)
        .send()
        .expect("Failed to fetch registration leaf");

    if !resp.status().is_success() {
        println!(
            "cargo:warning=Failed to fetch registration leaf for {} (status: {})",
            id,
            resp.status()
        );
        return;
    }

    let body = resp.text().expect("Failed to read response");
    let path = fixtures_dir.join(format!(
        "nuget/v3/registration5-semver1/{}/{}.json",
        id_lower, version
    ));
    fs::create_dir_all(path.parent().unwrap()).unwrap();
    fs::write(&path, &body).expect("Failed to write fixture");

    // Parse catalog URL and fetch catalog entry
    let leaf: serde_json::Value =
        serde_json::from_str(&body).expect("Failed to parse registration leaf JSON");
    if let Some(catalog_url) = leaf["catalogEntry"].as_str() {
        let resp = client
            .get(catalog_url)
            .send()
            .expect("Failed to fetch catalog entry");
        let body = resp.text().expect("Failed to read response");
        // Extract the path from the catalog URL (after .org/)
        let catalog_path = catalog_url.split(".org/").nth(1).unwrap_or("catalog.json");
        let path = fixtures_dir.join(format!("nuget/{}.json", catalog_path));
        fs::create_dir_all(path.parent().unwrap()).unwrap();
        fs::write(&path, &body).expect("Failed to write fixture");
    }

    // Fetch readme
    let url = format!(
        "https://api.nuget.org/v3-flatcontainer/{}/{}/readme",
        id_lower, version
    );
    if let Ok(resp) = client.get(&url).send()
        && resp.status().is_success()
    {
        let body = resp.text().unwrap();
        let path = fixtures_dir.join(format!(
            "nuget/v3-flatcontainer/{}/{}/readme.md",
            id_lower, version
        ));
        fs::create_dir_all(path.parent().unwrap()).unwrap();
        fs::write(&path, &body).expect("Failed to write fixture");
    }
}