undocumented-surface

#!/usr/bin/env python3
"""Does the documentation know about what this release contains?

Attach ABOVE any publish trigger — order matters within a game_loop event, and
a check that runs after the release has already been blessed is a report, not a
gate.

    .game_loop/triggers.json
    {"stepback": [{"name": "undocumented-surface",
                   "command": "/path/to/llm_chat/triggers/undocumented-surface"},
                  {"name": "<your-release-step>", "command": "..."}]}

THE FAILURE IT CATCHES is quiet and universal: a feature lands, its reasoning
goes into a commit message nobody will read again, and the two files a human or
an agent actually STARTS from never hear about it. This repo shipped a week of
user-facing surface that way — `--to`, `--json`, `mode`, `sync`, `pending`,
`briefing` — while writing careful paragraphs about each one in commits.

THE CHECK IS THE UNGAMEABLE HALF and deliberately narrow: every subcommand and
every option the CODE defines, against the documentation files. Prose cannot
satisfy it — the name appears or it does not.

WHAT IT CANNOT CHECK is the larger half, and this says so rather than letting a
green run imply otherwise: whether the docs are correct, whether they EXPLAIN a
thing or merely name it, or whether they still describe what the code does. A
mention is the floor, not the goal. The idea and the shape of that disclaimer
are game_loop's; this is their example grown into something testable.

Exit 0 always. A documentation gap must not block a retro — but it is loud, and
it names each gap, because a count nobody can act on is decoration.
"""
import argparse
import ast
import os
import re
import subprocess
import sys

OPTION = re.compile(r"""add_argument\(\s*["'](--[a-z][a-z0-9-]*)["']""")
VERB = re.compile(r"""add_parser\(\s*["']([a-z][a-z0-9_-]*)["']""")


def declared(paths):
    """(verbs, options) the code actually defines.

    Verbs come from `--help` when the tool can be run, because argparse is the
    only thing that knows for certain. The regex below misses any subcommand
    registered through a variable — this project registers `open` and `join`
    in a loop, so both were absent from the denominator, and the reverse walk
    duly reported two REAL commands as ghosts. A parser that misreads the
    source produces confident nonsense in whichever direction it is pointed.

    The regex stays as the fallback for a tool that cannot be executed here.
    """
    verbs, options = set(), set()
    for path in paths:
        try:
            with open(path) as f:
                source = f.read()
        except OSError:
            continue
        verbs.update(VERB.findall(source))
        options.update(OPTION.findall(source))
        verbs |= verbs_from_help(path)
    return verbs, options


def verbs_from_help(path, verb=None):
    """Subcommands as argparse itself lists them, or an empty set."""
    argv = [sys.executable, path] + ([verb] if verb else []) + ["--help"]
    try:
        done = subprocess.run(argv, capture_output=True, text=True, timeout=15)
    except Exception:
        return set()
    # A FLAG'S choices look identical to a subparser group, and taking the
    # first `{...}` conflates them. A sibling tool had `close` offering
    # {holds,partial,refuted,unverifiable} as flag values, and its lookup
    # reported the ordinary command `close mytask` as a ghost — a false
    # positive that had never fired, so a full green run said nothing.
    #
    # Their discriminator: flag choices are always attached to their flag, a
    # subparser group never is. Fixed here before it fires, because an
    # untriggered false positive is invisible in exactly the way a false
    # negative is — they are the same risk until something trips them.
    text = done.stdout or ""
    for found in re.finditer(r"\{([a-z0-9_,-]+)\}", text):
        before = text[:found.start()].rstrip()
        # `[` and `(` count as separators: argparse writes an optional flag as
        # `[--to {a,b}]`, so requiring whitespace before the flag made the
        # guard silently never match — the shape it exists for is exactly the
        # one it could not see.
        if re.search(r"(^|[\s\[(])--?[a-z][a-z0-9-]*$", before):
            continue          # these belong to the flag, not to the verb
        return set(found.group(1).split(","))
    return set()


def nested(path, verbs):
    """Verbs that have subcommands of their own.

    THIS CHECK ONLY VALIDATES THE FIRST WORD, and that is fine for a flat CLI
    and silently wrong for a nested one. A sibling agent found exactly this in
    their tool: `<tool> lock run` appeared in eight remedies, the check
    validated `lock` and stopped, and `lock` has five subcommands — so renaming
    `run` would kill all eight while the suite stayed green.

    A stated limit needs a corpse next to it or it reads as boilerplate, and
    this one can do better than a sentence: it can notice the day it starts
    mattering. This tool is flat today. If it ever is not, the check says so
    rather than quietly validating half of every remedy.
    """
    return sorted(v for v in verbs if verbs_from_help(path, v))


def is_the_tool(word, tool):
    """Is this token an invocation of the CLI, or something else ending in it?

    `endswith` matched `@llm_chat`, which is the SLACK BRIDGE — a different
    surface with its own verbs, deliberately not CLI subcommands. So
    `@llm_chat list` and the typo example `@llm_chat lsit` were both reported
    as commands that do not exist, which is true and beside the point: they
    were never claimed to be CLI commands.

    Two false positives standing in a check's output is not a cosmetic
    problem. It is the whole problem — nobody acts on a list that is mostly
    noise, and a check nobody acts on has stopped being a check. Both of them
    had been there long enough to be part of the furniture.

    The tool is the bare name, or a path ending in it. `@name`, `x-name` and
    `name-mcp` are not.

    FORMATTING IS STRIPPED FIRST, and leaving that out was a REGRESSION I
    introduced and then caught by sweeping the predicate rather than trusting
    the three ghosts it had removed. `endswith` accepted a backticked
    "`llm_chat" because it ends with the name; the exact match rejected it —
    so tightening the rule silently blinded the ghost check to the most common
    way this repo writes a remedy in a source string. Two false positives
    removed, a whole family of true ones lost.

    `@` IS NOT FORMATTING and is deliberately absent from the strip set: it is
    what distinguishes the Slack bridge from the CLI, which is the whole point
    of this function. Nor is `-` or `/`.

    lamp-owner's rule, arriving the hour I needed it: when you find a defect by
    probing, sweep the predicate before declaring it fixed. The probe tells you
    what is wrong, never what your fix also changed.
    """
    bare = word.strip("`'\"*()[]\\,.:;")
    return bare == tool or bare.endswith("/" + tool)


def named_in_strings(path, tool):
    """Commands named in a module's STRING LITERALS — not its docstrings.

    THE SCAN HAD A DENOMINATOR AND NOBODY ASKED FOR IT. Matching only
    backticks meant fifteen real verbs in this file — every "run this instead"
    remedy — sat where the ghost check could not see them. They were not
    failing. They were ABSENT, and absent reads exactly like correct: rename
    one of those verbs and every remedy naming it becomes a dead command the
    check is structurally blind to. The question came from the sibling agent
    who went looking for what their own rule could not see and found six.

    Literals, not docstrings, and that distinction does two jobs. Remedy text
    is a literal; explanatory prose is a docstring. So this sees every printed
    instruction, and it CANNOT see the sentence "a consumer vendored llm_chat
    instead of pointing at a sibling clone" that produced a false positive when
    the rule was positional. The discriminator was never the file extension —
    it is what a string is FOR, which is a thing the AST already knows.
    """
    try:
        with open(path) as f:
            tree = ast.parse(f.read())
    except (OSError, SyntaxError, ValueError):
        return set()
    docstrings = set()
    for node in ast.walk(tree):
        if isinstance(node, (ast.Module, ast.FunctionDef, ast.AsyncFunctionDef,
                             ast.ClassDef)):
            doc = ast.get_docstring(node, clean=False)
            if doc is not None:
                docstrings.add(doc)
    found = set()
    for node in ast.walk(tree):
        if not isinstance(node, ast.Constant) or not isinstance(node.value, str):
            continue
        if node.value in docstrings:
            continue
        # COMMAND POSITION, not mere mention. A remedy sits at the start of its
        # own line; prose does not. Without this, "no llm_chat server at {url}"
        # and "this llm_chat checkout" were reported as ghost commands —
        # matching anywhere inside a literal cannot tell an instruction from a
        # sentence that happens to contain the tool's name.
        for line in node.value.splitlines():
            words = line.strip().split()
            if len(words) >= 2 and is_the_tool(words[0].rstrip(":"), tool):
                if re.fullmatch(r"[a-z][a-z0-9_-]*", words[1]):
                    found.add(words[1])
    return found


def assembled_remedies(path, tool):
    """How many remedies are built across f-string pieces, and so unchecked.

    A COUNT, not a caveat, and that is the whole point. "f-string remedies are
    not validated" is a true sentence that keeps printing after the last such
    remedy is deleted — at which point it is the decoration this project
    already learned to distrust. And in the other direction, the first
    assembled remedy somebody writes announces nothing at all.

    Counting makes the claim expire on its own. Zero prints nothing; one prints
    one. The number IS the announcement, which is what a stated limit could
    never be — a sibling agent and I spent seven rounds on limits that stayed
    accurate and were read as boilerplate, and neither of us could see an
    instrument for the transition. This is that instrument, for this one case.
    """
    try:
        with open(path) as f:
            tree = ast.parse(f.read())
    except (OSError, SyntaxError, ValueError):
        return 0
    count = 0
    for node in ast.walk(tree):
        if not isinstance(node, ast.JoinedStr):
            continue
        literal = "".join(part.value for part in node.values
                          if isinstance(part, ast.Constant)
                          and isinstance(part.value, str))
        # COMMAND POSITION, not mention — the same discriminator the rest of
        # this file needed, and I got it wrong here after getting it right
        # twice elsewhere. "MENTIONS the tool" counted `You have been invited
        # to an llm_chat channel:` and `no llm_chat server at {x} — start one
        # with ./zonai serve`, which are prose and a remedy for a DIFFERENT
        # tool. Twelve, of which five were the subject.
        #
        # A sibling agent hit the mirror image within minutes of taking this
        # idea: their subject was too wide, then their fix made it too narrow
        # and let through the shape the check is named for. Both counts were
        # green and neither number showed which population it had measured.
        if not any(isinstance(part, ast.FormattedValue)
                   for part in node.values):
            continue
        for line in literal.splitlines():
            stripped = line.strip()
            if (stripped.split(" ")[0].rstrip(":").endswith(tool)
                    and " " in stripped) or "`%s " % tool in line:
                count += 1
                break
    return count


def bare_words(line, tool):
    """Literal words in a remedy after the verb — not placeholders or flags.

    `llm_chat mode {name} ordinary --yes` yields ("mode", ["ordinary"]).
    `{name}` is filled in at runtime and `--yes` is an option; neither can go
    stale the way a hard-coded VALUE can.
    """
    words = line.strip().split()
    if len(words) < 2 or not words[0].rstrip(":").endswith(tool):
        return None, []
    rest = []
    for word in words[2:]:
        word = word.strip(".,;)")
        if word.startswith(("-", "{", "<", "$")) or not word:
            continue
        if re.fullmatch(r"[a-z][a-z0-9_-]*", word):
            rest.append(word)
    return words[1], rest


def stale_values(path, tool, choices):
    """Hard-coded VALUES in remedies that are no longer valid choices.

    The first word was validated and the check stopped there. A sibling agent
    found eight remedies naming `<tool> lock run` where `lock` was checked and
    `run` — one of five subcommands — was not, so renaming it would kill all
    eight silently.

    This repo had the same exposure through `choices=`: `mode` accepts
    broadcast|ordinary and its own reversal remedy hard-codes one of them.
    Argparse renders a choices positional exactly like a subparser group, so I
    first read the warning as a false positive and nearly dismissed it. It was
    a true one wearing an unfamiliar shape.
    """
    try:
        with open(path) as f:
            tree = ast.parse(f.read())
    except (OSError, SyntaxError, ValueError):
        return []
    stale = []
    for node in ast.walk(tree):
        if not isinstance(node, ast.Constant) or not isinstance(node.value, str):
            continue
        for line in node.value.splitlines():
            verb, rest = bare_words(line, tool)
            if verb not in choices or not rest:
                continue
            # AT LEAST ONE valid choice, rather than "every word is valid".
            # Which POSITION holds the value cannot be known generally —
            # `llm_chat mode <channel> <mode>` puts a room name in between, and
            # flagging every non-choice word reported the room. Asking whether
            # the remedy names any accepted value at all needs no position and
            # cannot be confused by an argument that is not a value.
            if not any(word in choices[verb] for word in rest):
                stale.append((verb, " ".join(rest)))
    return sorted(set(stale))


def invented(verbs, sources, docs, tool):
    """Commands that are NAMED but do not exist. The reverse walk.

    The conventional check has a DIRECTION — it walks real commands asking "is
    each mentioned?" — and that direction cannot catch a remedy naming a verb
    the parser rejects. Pointed out by a sibling agent who then found a live
    one in their own repo: a refusal message ending "track it in the campaign
    record (`showrunner campaign`)", with no `campaign` verb. Argparse refuses
    it outright.

    That is the worst place for it. The only route to a refusal string is being
    blocked already, so the reader is the one person least able to route around
    a wrong instruction — and it can never be found by use, because nobody who
    is working ever sees it.

    A COMMAND IS WHAT APPEARS IN BACKTICKS OR A FENCED BLOCK, which is their
    implementation note and worth taking whole. Their first version told
    commands from prose with a denylist of English words and grew by eight
    entries on its first run, because a denylist tracks the LANGUAGE rather
    than the code and would grow forever. A positional rule stays fixed as the
    docs grow.
    """
    named = set()
    for path in sources:
        named |= named_in_strings(path, tool)
    for path in docs:
        try:
            with open(path) as f:
                text = f.read()
        except OSError:
            continue
        for quoted in (re.findall(r"`([^`\n]+)`", text)
                       + re.findall(r"^\s{4}(\S.*)$", text, re.M)):
            words = quoted.replace("./", "").split()
            for i, word in enumerate(words[:-1]):
                if is_the_tool(word, tool) and words[i + 1].isidentifier():
                    named.add(words[i + 1])
    return sorted(n for n in named if n not in verbs)


def undocumented(names, docs):
    """Names that appear in NO documentation file.

    Substring, not word-boundary: `--to` legitimately appears inside `--to-all`
    in prose, and demanding a standalone mention would report a gap that is not
    there. This check errs toward silence — a false alarm here trains people to
    ignore it, and an ignored check is worse than none.
    """
    text = ""
    for path in docs:
        try:
            with open(path) as f:
                text += f.read()
        except OSError:
            continue
    return sorted(n for n in names if n not in text)


def unmentioned_entrypoints(repo, bindir, agent_doc):
    """Executables in bin/ that the AGENT-facing doc never names.

    Every other check here pools the docs: a name found in README.md counts as
    documented and llms.txt is never asked about it separately. For flags that
    is right — spelling every option out twice is noise nobody reads.

    For an ENTRYPOINT it is the blind spot. `bin/llm-chat-mcp` shipped with
    three mentions in README.md and none in llms.txt, so an entire integration
    surface — the one that puts these verbs in an agent's tool list — was
    invisible to the readers the file exists for, while every name-level check
    stayed green. Pooling made the gap unreportable: the name WAS documented,
    just not anywhere its audience starts.

    So this one file is asked on its own. It is deliberately about existence,
    not quality: a surface an agent cannot discover is worse than one
    described badly, because the second at least gets read.
    """
    try:
        with open(os.path.join(repo, agent_doc)) as f:
            text = f.read()
    except OSError:
        return []                     # a missing agent doc is reported elsewhere
    try:
        entries = sorted(os.listdir(os.path.join(repo, bindir)))
    except OSError:
        return []
    missing = []
    for name in entries:
        path = os.path.join(repo, bindir, name)
        if name.startswith(".") or name.startswith("__") or os.path.isdir(path):
            continue
        if not os.access(path, os.X_OK):
            continue
        if name not in text:
            missing.append(name)
    return missing


def main(argv=None):
    ap = argparse.ArgumentParser()
    ap.add_argument("--source", nargs="+", default=["bin/llm_chat"])
    ap.add_argument("--docs", nargs="+", default=["README.md", "llms.txt"])
    ap.add_argument("--bindir", default="bin")
    ap.add_argument("--agent-doc", dest="agent_doc", default="llms.txt",
                    help="the doc an AGENT starts from, asked about "
                         "entrypoints on its own rather than pooled")
    ap.add_argument("--repo", default=None)
    ap.add_argument("--tool", default="llm_chat",
                    help="the command name that precedes a subcommand")
    args = ap.parse_args(argv)

    if not sys.stdin.isatty():
        sys.stdin.read()          # drain the payload; nothing here needs it

    repo = args.repo or os.environ.get("GAME_LOOP_REPO") or os.getcwd()
    verbs, options = declared([os.path.join(repo, p) for p in args.source])
    docs = [os.path.join(repo, p) for p in args.docs]

    missing_verbs = undocumented(verbs, docs)
    missing_options = undocumented(options, docs)
    ghosts = invented(verbs, [os.path.join(repo, p) for p in args.source],
                      docs, args.tool)
    if ghosts:
        print("NAMED BUT NOT REAL — these are printed or documented as "
              "commands and do not exist:")
        for ghost in ghosts:
            print("  %s %s" % (args.tool, ghost))
        print("\n  The reverse walk. A check that asks 'is every real command "
              "mentioned?'\n  cannot see these. A remedy naming a verb the "
              "parser rejects is handed to\n  the one reader least able to "
              "route around it, and can never be found by\n  use, because "
              "nobody who is working ever sees a refusal.\n")

    stale = []
    for path in [os.path.join(repo, p) for p in args.source]:
        choices = {v: verbs_from_help(path, v) for v in verbs}
        choices = {v: c for v, c in choices.items() if c}
        stale += stale_values(path, args.tool, choices)
    if stale:
        print("SECOND-WORD DRIFT — a remedy hard-codes a value that is no "
              "longer accepted:")
        for verb, word in stale:
            print("  %s %s ... %s" % (args.tool, verb, word))
        print("\n  Validating only the first word leaves these half-checked: "
              "rename the VALUE\n  and every remedy naming it goes dead while "
              "this check stays green. A sibling\n  tool had eight remedies in "
              "exactly that state.\n")

    # COVERAGE, stated whether or not anything was found. This validates second
    # words only where the whole remedy is ONE literal. This project's own
    # `mode` reversal is assembled from f-string pieces —
    #   f"...llm_chat mode {name} " + f"{'ordinary' if want else 'broadcast'}"
    # — so the value lives in a different AST node from the verb and cannot be
    # matched. That remedy is real, hard-coded, and unchecked.
    #
    # Said out loud because a clean run right after tightening a check is when
    # it is most likely to have become silence, and because a limit needs a
    # worked example beside it or it reads as boilerplate. This is the example.
    for path in [os.path.join(repo, p) for p in args.source]:
        deep = nested(path, verbs)
        assembled = assembled_remedies(path, args.tool)
        if deep and assembled:
            print("PARTLY CHECKED — these verbs take a second word: %s"
                  % ", ".join(deep))
            print("  %d remedies here are assembled across f-string pieces, "
                  "so the verb and\n  its value sit in different nodes and "
                  "cannot be paired. Those are NOT\n  validated. This line "
                  "retires itself when that count reaches zero — the\n  count "
                  "is the announcement, which a fixed caveat could never be.\n"
                  % assembled)

    strangers = unmentioned_entrypoints(repo, args.bindir, args.agent_doc)
    if strangers:
        print("NOT IN %s — a shipped entrypoint the agent-facing doc never "
              "names:" % args.agent_doc)
        for name in strangers:
            print("  %s/%s" % (args.bindir, name))
        print("\n  Every other check here pools the docs, so a mention in "
              "README.md counts and\n  this file is never asked separately. "
              "That is how an entire integration\n  surface shipped "
              "documented and undiscoverable at once.\n")

    if (not missing_verbs and not missing_options and not ghosts and not stale
            and not strangers):
        print("every command and option is named in the docs "
              "(which is the floor, not the goal — it cannot tell you whether "
              "they are correct or merely mentioned)")
        return 0

    # ONLY WHEN THERE IS SOMETHING UNDER IT. This heading printed
    # unconditionally on the failure path, so a run whose only finding was a
    # GHOST announced "UNDOCUMENTED SURFACE — shipped, and absent from
    # README.md, llms.txt:" followed by nothing at all. A heading with no
    # findings under it is itself a false finding, in a report whose entire
    # job is to be believed — and it appeared the moment the option list was
    # cleared, which is exactly when somebody is checking whether they are
    # done.
    if missing_verbs or missing_options:
        print("UNDOCUMENTED SURFACE — shipped, and absent from %s:"
              % ", ".join(args.docs))
        for verb in missing_verbs:
            print("  command  %s" % verb)
        for option in missing_options:
            print("  option   %s" % option)
        print("\n  A reader starts from those files. Anything not in them "
              "exists only for\n  whoever already knew to look. This checks "
              "that a name APPEARS — never\n  that the prose is right, or "
              "that it still describes what the code does.")
    return 0


if __name__ == "__main__":
    sys.exit(main())