diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ceea71cf4..d283a829ce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,10 @@ All notable changes to the DKG V10 node are documented here. The format is based ## [Unreleased] +### Removed — `oxigraph-worker` backend support + +- **The embedded `oxigraph-worker` backend is retired.** The storage package no longer exports or registers the worker-thread adapter, `createTripleStore({ backend: "oxigraph-worker" })` fails with an actionable migration error, and CLI config validation refuses explicit `store.backend: "oxigraph-worker"` before daemon boot. Block-less configs now resolve to the daemon-managed `oxigraph-server` default; if a legacy `store.nq` file exists, daemon boot requires `DKG_ACCEPT_STORE_RESET=1` so operators acknowledge the fresh-store cutover. + ## [10.0.6] - 2026-07-10 Sync-, storage-, and admission-path hardening on top of 10.0.5, plus the StorageACK priority-lane follow-ups and a set of CLI/RPC fixes. Eliminates the perpetually-dirty graph-set-index full scan that was saturating managed Oxigraph cores, bounds sync-responder memory and coalesces duplicate sync fan-out, makes network admission probing back off and use canonical peer ids, and completes the StorageACK priority-lane hardening (ACK candidate selection, async promote-queue read serialization, and OT-RFC-49 host-mode ciphertext strip-by-curation). **No smart-contract changes — no deployment required** (no Solidity source, ABI, or mainnet/testnet deployment-registry changes since 10.0.5). diff --git a/README.md b/README.md index cb9a5aa15e..8fb7d724a0 100644 --- a/README.md +++ b/README.md @@ -403,11 +403,13 @@ analysis reports are under `bench/results/profiles/`, including ## Triple Store Backends -A DKG node keeps every assertion in an [RDF](https://www.w3.org/RDF/) triple store. Out of the box the node runs an embedded [Oxigraph](https://github.com/oxigraph/oxigraph) instance, which is everything you need on a workstation — no extra process, no extra port, no extra config. Heavier deployments can swap in [Blazegraph](https://blazegraph.com/) (the mainnet store) or any SPARQL 1.1 server. +A DKG node keeps every assertion in an [RDF](https://www.w3.org/RDF/) triple store. Out of the box the daemon manages a local [Oxigraph](https://github.com/oxigraph/oxigraph) server, so a workstation needs no separate setup. Heavier deployments can swap in [Blazegraph](https://blazegraph.com/) (the mainnet store) or any SPARQL 1.1 server. | Backend | When to pick it | |---|---| -| `oxigraph-worker` (default) | Single-operator nodes, dev, CI. No setup. File-backed, capped at process RAM. | +| `oxigraph-server` (default) | Single-operator nodes, dev, CI. Managed automatically by the daemon with persistent local storage. | +| `oxigraph` | Embedded in-memory Oxigraph for development and short-lived tests. | +| `oxigraph-persistent` | Embedded persistent Oxigraph when an explicit existing store path is required. | | `blazegraph` | High-throughput nodes, mainnet parity, very large graphs (10M+ quads). Run as a separate daemon (Docker or `java -jar`). Shares cleanly with V6 / V8 instances — DKG scopes its writes to the `did:dkg:context-graph:` named-graph prefix. | | `sparql-http` | Any SPARQL 1.1 Protocol server (Fuseki, GraphDB, Stardog, Neptune…). Bring your own URL + (optional) auth header. | @@ -420,7 +422,7 @@ Two paths: ``` $ dkg init … -Triple store backend (oxigraph / blazegraph) (oxigraph): blazegraph +Triple store backend (oxigraph-server / oxigraph / blazegraph) (oxigraph-server): blazegraph Blazegraph SPARQL endpoint URL: http://127.0.0.1:9999/bigdata/namespace/mynode/sparql Store endpoint reachable: blazegraph http://127.0.0.1:9999/bigdata/namespace/mynode/sparql ``` diff --git a/bench/store-read-latency.bench.ts b/bench/store-read-latency.bench.ts index 28d16edd52..9f2ec700ce 100644 --- a/bench/store-read-latency.bench.ts +++ b/bench/store-read-latency.bench.ts @@ -1,5 +1,3 @@ -import { existsSync } from 'node:fs'; -import { fileURLToPath } from 'node:url'; import { defineSuite } from 'esbench'; // Import the store classes + types from their SPECIFIC source modules rather // than the storage barrel: the barrel also re-exports GraphManager / @@ -8,7 +6,6 @@ import { defineSuite } from 'esbench'; // clean checkout. These adapter modules depend only on `oxigraph` + the local // triple-store types, so the bench needs nothing beyond the storage build. import { OxigraphStore } from '../packages/storage/src/adapters/oxigraph.ts'; -import { OxigraphWorkerStore } from '../packages/storage/src/adapters/oxigraph-worker.ts'; import type { Quad, QueryResult } from '../packages/storage/src/triple-store.ts'; import { GET_TOTAL_TRIPLES_SPARQL, parseRdfInt } from '../packages/cli/src/daemon/metrics-queries.ts'; import { benchAsyncWithHooks } from './support/esbench-case-hooks.ts'; @@ -24,33 +21,11 @@ import { benchAsyncWithHooks } from './support/esbench-case-hooks.ts'; * - the production `getTotalTriples` `COUNT(*)` aggregate the 30s metrics * collector runs (`packages/cli/src/daemon/lifecycle.ts`). * - * Backends (env `DKG_BENCH_STORE_BACKENDS`, comma-separated): - * - `inprocess` — `OxigraphStore`, no build step required. - * - `worker` — `OxigraphWorkerStore`, the PRODUCTION backend; requires - * `pnpm --filter @origintrail-official/dkg-storage build` first (it spawns a - * compiled worker artefact). - * Default: `inprocess`, plus `worker` automatically when its compiled artefact - * is present (so a built tree / CI measures both; an unbuilt tree degrades to - * `inprocess`-only instead of erroring). - * - * CONTENTION — the `(under write load)` cases — is measured ONLY on the `worker` - * backend, by design. The production read-starvation is a property of the - * single-writer oxigraph WORKER, whose message queue serialises a read behind - * queued writes — exactly what an out-of-process MVCC server (#938) relieves. - * The in-process `OxigraphStore` runs its insert/query work SYNCHRONOUSLY on one - * thread, so a read and a write can never truly overlap; a same-thread "writer" - * would only measure event-loop interleaving (not store contention) and could - * report misleadingly idle reads. So `inprocess` runs the idle read-latency - * baselines only. - * * Store sizes via env `DKG_BENCH_STORE_SIZES` (default `1k,50k`). */ -type Backend = 'inprocess' | 'worker'; - interface ReadStore { insert(quads: Quad[]): Promise; - delete(quads: Quad[]): Promise; query(sparql: string): Promise; close(): Promise; } @@ -63,13 +38,6 @@ const READ_LIMIT1 = `SELECT ?s WHERE { GRAPH <${GRAPH}> { ?s ?p ?o } } LIMIT 1`; // synthetic data lives in a named graph, so the `GRAPH ?g` branch carries the scan. const READ_TOTAL_TRIPLES = GET_TOTAL_TRIPLES_SPARQL; -// Bounded write churn: the background writer repeatedly inserts then deletes a -// fixed batch in a region disjoint from the pre-populated base, so it generates -// sustained write work WITHOUT drifting the store size (which would otherwise -// confound the getTotalTriples-under-load measurement). -const CHURN_BATCH = 50; -const CHURN_OFFSET = 1_000_000_000; - const STORE_SIZES: Record = { '1k': 1_000, '10k': 10_000, '50k': 50_000, '200k': 200_000 }; const INSERT_CHUNK = 1_000; @@ -111,46 +79,6 @@ function makeQuads(count: number, offset: number): Quad[] { return quads; } -// The compiled worker artefact sits next to the storage dist build; its presence -// means the `worker` backend can be constructed without throwing. -function workerArtifactAvailable(): boolean { - try { - return existsSync(fileURLToPath(new URL('../packages/storage/dist/adapters/oxigraph-worker-impl.js', import.meta.url))); - } catch { - return false; - } -} - -function resolveBackends(): Backend[] { - const raw = process.env.DKG_BENCH_STORE_BACKENDS?.trim(); - if (!raw) { - if (workerArtifactAvailable()) return ['inprocess', 'worker']; - // Loud, because the worker backend is the one this regression is about — a - // silent inprocess-only run would look like it measured the production path. - console.warn( - '[store-read-latency] worker backend SKIPPED (compiled adapter missing) — ' + - 'measuring `inprocess` idle read latency ONLY, NOT the production write-contention path. ' + - 'Run `pnpm --filter @origintrail-official/dkg-storage build` (or `pnpm bench:store-read`, ' + - 'which builds it) to include the worker backend.', - ); - return ['inprocess']; - } - const known = new Set(['inprocess', 'worker']); - const requested = raw.split(',').map((p) => p.trim().toLowerCase()).filter(Boolean); - for (const b of requested) { - if (!known.has(b as Backend)) { - throw new Error(`Unknown DKG_BENCH_STORE_BACKENDS entry "${b}". Expected: inprocess, worker`); - } - } - if (requested.includes('worker') && !workerArtifactAvailable()) { - throw new Error( - 'DKG_BENCH_STORE_BACKENDS requested "worker", but the compiled adapter is missing. ' + - 'Run `pnpm --filter @origintrail-official/dkg-storage build` first.', - ); - } - return requested.length > 0 ? (requested as Backend[]) : ['inprocess']; -} - function resolveStoreSizeLabels(): string[] { const raw = process.env.DKG_BENCH_STORE_SIZES?.trim(); const labels = raw ? raw.split(',').map((p) => p.trim().toLowerCase()).filter(Boolean) : ['1k', '50k']; @@ -162,15 +90,8 @@ function resolveStoreSizeLabels(): string[] { return labels; } -function createStore(backend: Backend): ReadStore { - // `worker` availability is gated in resolveBackends(), so by the time we get - // here the compiled adapter is present. - return backend === 'worker' ? new OxigraphWorkerStore() : new OxigraphStore(); -} - export default defineSuite({ params: { - backend: resolveBackends(), storeSize: resolveStoreSizeLabels(), }, baseline: { @@ -185,63 +106,20 @@ export default defineSuite({ warmup: 1, }, async setup(scene) { - const backend = scene.params.backend as Backend; const sizeLabel = scene.params.storeSize as string; const quadCount = STORE_SIZES[sizeLabel]; - const store = createStore(backend); + const store: ReadStore = new OxigraphStore(); - const churn = makeQuads(CHURN_BATCH, CHURN_OFFSET); - let writerActive = false; - let writerDone: Promise | undefined; - let writerError: unknown; - - const stopWriter = async (): Promise => { - if (!writerDone) return; - writerActive = false; - const done = writerDone; - writerDone = undefined; - // The loop captures its own failures into `writerError`, so this never rejects. - await done; - // Remove the churn batch so a half-applied cycle (an insert without its - // matching delete) can't leak into the next iteration's getTotalTriples count. - try { - await store.delete(churn); - } catch { - /* store may be mid-teardown */ - } - // Fail fast if the writer died: a stopped writer must never let an - // `(under write load)` case record a successful (effectively idle) sample. - if (writerError !== undefined) { - const err = writerError; - writerError = undefined; - throw new Error(`background writer died during the under-load iteration: ${errorText(err)}`); - } - }; - - // ONE ordered teardown for the scene: stop the writer and await its loop - // BEFORE closing the store, so an in-flight insert/delete can never race a - // closed store/worker. esbench may run scene teardown hooks concurrently, so - // all ordering lives inside this single callback. Registered up-front so the - // store is still closed (and any worker thread terminated) even if the - // population below throws. + // Registered up-front so the store is still closed even if population or a + // benchmark case throws. scene.teardown(async () => { - let stopErr: unknown; - try { - await stopWriter(); - } catch (err) { - stopErr = err; // capture; still close the store below - } - // `close()` is the only place the worker thread is joined, so a failure - // here is a real teardown bug — surface it (log + reject) rather than - // swallow it and let a broken worker benchmark look green. try { await store.close(); } catch (closeErr) { - console.error(`[store-read-latency] store.close() failed (worker thread may not have joined): ${errorText(closeErr)}`); + console.error(`[store-read-latency] store.close() failed: ${errorText(closeErr)}`); throw closeErr; } - if (stopErr !== undefined) throw stopErr; }); // Pre-populate the base graph the reads scan. @@ -256,63 +134,5 @@ export default defineSuite({ benchAsyncWithHooks(scene, 'read getTotalTriples (idle)', async () => { assertCountAtLeast(await store.query(READ_TOTAL_TRIPLES), quadCount, 'read getTotalTriples'); }, {}); - - // Contention is meaningful only on the worker backend (see file docstring): - // the in-process store is single-threaded + synchronous, so reads and writes - // cannot truly overlap. Skip the `(under write load)` cases for it rather - // than report a misleading same-thread number. - if (backend !== 'worker') return; - - // Background writer for the worker-backend `(under write load)` cases, - // scoped to each loaded iteration (not to case-execution order): - // - `beforeIteration` (startWriter) AWAITS one full insert/delete cycle so - // writes are provably queued in the worker before the measured read. - // - a writer that dies is recorded in `writerError` and surfaced as a - // FAILED case — startWriter throws if it died before the first cycle, - // the workload throws if it died mid-measurement, and stopWriter throws - // if it died just after — so a broken writer can never silently degrade - // into an idle read. - // - `afterIteration` (stopWriter) stops it and clears the churn batch. - const startWriter = async (): Promise => { - if (writerActive) return; - writerActive = true; - writerError = undefined; - let signalFirstCycle!: () => void; - const firstCycle = new Promise((resolve) => { signalFirstCycle = resolve; }); - writerDone = (async () => { - try { - let signalled = false; - while (writerActive) { - await store.insert(churn); - await store.delete(churn); - if (!signalled) { signalled = true; signalFirstCycle(); } - } - } catch (err) { - writerError = err; - } finally { - // Unblock the barrier even if the first cycle threw, so a writer - // failure surfaces (below) instead of hanging the benchmark. - signalFirstCycle(); - } - })(); - await firstCycle; - if (writerError !== undefined) { - throw new Error(`background writer failed before its first write cycle: ${errorText(writerError)}`); - } - }; - - benchAsyncWithHooks(scene, 'read LIMIT 1 (under write load)', async () => { - assertNonEmptySelect(await store.query(READ_LIMIT1), 'read LIMIT 1'); - if (writerError !== undefined) { - throw new Error(`background writer died during the measurement: ${errorText(writerError)}`); - } - }, { beforeIteration: startWriter, afterIteration: stopWriter }); - - benchAsyncWithHooks(scene, 'read getTotalTriples (under write load)', async () => { - assertCountAtLeast(await store.query(READ_TOTAL_TRIPLES), quadCount, 'read getTotalTriples'); - if (writerError !== undefined) { - throw new Error(`background writer died during the measurement: ${errorText(writerError)}`); - } - }, { beforeIteration: startWriter, afterIteration: stopWriter }); }, }); diff --git a/docs/use-dkg/storage-sparql-http.md b/docs/use-dkg/storage-sparql-http.md index 9d70af2f93..3a8ea2c5b8 100644 --- a/docs/use-dkg/storage-sparql-http.md +++ b/docs/use-dkg/storage-sparql-http.md @@ -7,7 +7,7 @@ doc_type: how-to # Using an external SPARQL store (Oxigraph server, etc.) -The DKG node can use any **SPARQL 1.1 Protocol**–compliant store you run yourself, instead of its default daemon-managed local Oxigraph server (or the embedded `oxigraph-worker` fallback). That gives you: +The DKG node can use any **SPARQL 1.1 Protocol**–compliant store you run yourself, instead of its default daemon-managed local Oxigraph server. That gives you: - **Real on-disk persistence** (e.g. Oxigraph server with RocksDB) - **Larger graphs** without holding everything in the Node process @@ -118,4 +118,4 @@ await agent.start(); New installs default to a **daemon-managed local Oxigraph server** (`store.backend: "oxigraph-server"`): `dkg init`, `dkg openclaw/hermes/mcp setup`, or accepting the wizard default writes this block. The daemon fetches the pinned `oxigraph` binary on first boot and runs it on loopback, giving MVCC concurrent reads and incremental RocksDB persistence. -If a config has **no** `store` block at all, the runtime falls back to the embedded in-process **`oxigraph-worker`** (a single-writer store that rewrites its on-disk N-Quads dump under `dataDir` on every flush) — fine for development and small nodes. For very large graphs or existing infrastructure, use `sparql-http` with an external store. +If a config has **no** `store` block at all, the runtime now uses the same daemon-managed **`oxigraph-server`** default. The old embedded **`oxigraph-worker`** backend has been retired; configs that still name it fail fast with a migration message. For very large graphs or existing infrastructure, use `sparql-http` with an external store. diff --git a/packages/agent/src/dkg-agent-helpers.ts b/packages/agent/src/dkg-agent-helpers.ts index 3d53b4157a..5e7a39b814 100644 --- a/packages/agent/src/dkg-agent-helpers.ts +++ b/packages/agent/src/dkg-agent-helpers.ts @@ -403,7 +403,6 @@ export function applyDefaultLargeLiteralStorage( export function isLocalOxigraphConfig(storeConfig: TripleStoreConfig): boolean { return storeConfig.backend === 'oxigraph' - || storeConfig.backend === 'oxigraph-worker' || storeConfig.backend === 'oxigraph-persistent'; } diff --git a/packages/agent/src/dkg-agent-types.ts b/packages/agent/src/dkg-agent-types.ts index 7b3825d16e..a64cd160a7 100644 --- a/packages/agent/src/dkg-agent-types.ts +++ b/packages/agent/src/dkg-agent-types.ts @@ -923,7 +923,7 @@ export interface DKGAgentConfig { }>; dataDir?: string; store?: TripleStore; - /** Triple store backend configuration (e.g. oxigraph-worker, blazegraph). If omitted, defaults to oxigraph-worker when dataDir is set. */ + /** Triple store backend configuration (e.g. oxigraph-server runtime view, oxigraph-persistent, blazegraph). If omitted, dataDir agents use oxigraph-persistent. */ storeConfig?: TripleStoreConfig; /** Out-of-line storage for large public SWM RDF literal object terms. Defaults on for local Oxigraph-backed dataDir stores. */ largeLiteralStorage?: LargeLiteralStorageConfig; diff --git a/packages/agent/src/dkg-agent.ts b/packages/agent/src/dkg-agent.ts index f27f95994a..7d360f6c8b 100644 --- a/packages/agent/src/dkg-agent.ts +++ b/packages/agent/src/dkg-agent.ts @@ -685,11 +685,11 @@ export class DKGAgent extends DKGAgentBase { const { join } = await import('node:path'); const persistPath = join(config.dataDir, 'store.nq'); store = await createTripleStore({ - backend: 'oxigraph-worker', + backend: 'oxigraph-persistent', options: { path: persistPath }, largeLiteralStorage: defaultLargeLiteralStorage(config.dataDir, config.largeLiteralStorage), }); - log.info(ctx, `Persistent triple store (worker thread): ${persistPath}`); + log.info(ctx, `Persistent triple store: ${persistPath}`); } else { store = await createTripleStore({ backend: 'oxigraph' }); log.warn(ctx, `No dataDir — triple store is in-memory (data will be lost on restart)`); diff --git a/packages/agent/src/sync/responder/sync-handler.ts b/packages/agent/src/sync/responder/sync-handler.ts index 646afe3515..1ea7142003 100644 --- a/packages/agent/src/sync/responder/sync-handler.ts +++ b/packages/agent/src/sync/responder/sync-handler.ts @@ -447,7 +447,7 @@ export function registerSyncHandler(params: RegisterSyncHandlerParams): void { warnedPreDispatchCancellation = true; logWarn( createOperationContext('sync'), - 'Sync responder is using a store backend whose query AbortSignal is pre-dispatch only; in-flight sync queries cannot release responder capacity until the synchronous store call returns. Use oxigraph-worker or an HTTP SPARQL backend for interruptible long-query cancellation.', + 'Sync responder is using a store backend whose query AbortSignal is pre-dispatch only; in-flight sync queries cannot release responder capacity until the synchronous store call returns. Use an HTTP SPARQL backend for interruptible long-query cancellation.', ); } if (isWorkspace) { diff --git a/packages/agent/test/context-graph-discovery.test.ts b/packages/agent/test/context-graph-discovery.test.ts index bda11d560d..33429c681e 100644 --- a/packages/agent/test/context-graph-discovery.test.ts +++ b/packages/agent/test/context-graph-discovery.test.ts @@ -1303,8 +1303,10 @@ describe('listContextGraphs merge', () => { }, 15000); it('bypasses list cache for unknown configured store backends', async () => { - const backend = 'test-remote-list-cache-backend'; - registerTripleStoreAdapter(backend, async () => new OxigraphStore()); + const backend = registerTripleStoreAdapter( + 'test-remote-list-cache-backend', + async () => new OxigraphStore(), + ); const created = await DKGAgent.create({ kaNumberAllocator: makeTestKaNumberAllocator(), name: 'ContextGraphTestAgent', diff --git a/packages/cli/src/commands/hermes.ts b/packages/cli/src/commands/hermes.ts index 1d95782816..7f98eb0359 100644 --- a/packages/cli/src/commands/hermes.ts +++ b/packages/cli/src/commands/hermes.ts @@ -32,6 +32,7 @@ import { import { ApiClient } from '../api-client.js'; import { parsePositiveIntegerOption, parsePositiveMsOption } from '../cli-option-parsers.js'; import { promptStoreBackend, applyStoreFlagsToConfig } from '../store-wizard.js'; +import { storeFlagBackendList } from '../store-backends.js'; import { runConfiguredSourceWorker } from '../source-worker-runner.js'; import { batchEntityQuads } from '../batching.js'; import { @@ -189,7 +190,7 @@ hermesCmd ) .option( '--store ', - 'Triple-store backend (oxigraph | blazegraph | sparql-http). Validates the URL and persists the store block after setup.', + `Triple-store backend (${storeFlagBackendList(' | ')}). Validates the URL and persists the store block after setup.`, ) .option( '--store-url ', diff --git a/packages/cli/src/commands/init.ts b/packages/cli/src/commands/init.ts index 4bd216b3fe..d49ad22b9b 100644 --- a/packages/cli/src/commands/init.ts +++ b/packages/cli/src/commands/init.ts @@ -36,6 +36,7 @@ import { import { ApiClient } from '../api-client.js'; import { parsePositiveIntegerOption, parsePositiveMsOption } from '../cli-option-parsers.js'; import { promptStoreBackend, applyStoreFlagsToConfig } from '../store-wizard.js'; +import { storeFlagBackendList } from '../store-backends.js'; import { runConfiguredSourceWorker } from '../source-worker-runner.js'; import { batchEntityQuads } from '../batching.js'; import { @@ -216,7 +217,7 @@ program ) .option( '--store ', - 'Pre-fill the triple-store backend prompt (oxigraph | blazegraph | sparql-http).', + `Pre-fill the triple-store backend prompt (${storeFlagBackendList(' | ')}).`, ) .option( '--store-url ', @@ -534,9 +535,8 @@ program chain: isNetworkSwitch ? chainSection : (chainSection ?? existing.chain), auth: { enabled: enableAuth, tokens: existing.auth?.tokens }, // Persist the chosen backend. `storeBlock === null` from the - // wizard means "use the local default" — we explicitly clear any - // existing block so re-running `dkg init` to switch from - // blazegraph back to oxigraph actually applies. + // wizard means "leave the store block omitted"; daemon boot treats + // that as the managed `oxigraph-server` default. store: storeBlock ?? undefined, }; await saveConfig(config); @@ -573,7 +573,7 @@ program const endpoint = o?.url ?? o?.queryEndpoint; return `${storeBlock.backend}${endpoint ? ` (${endpoint})` : ''}`; })() - : 'oxigraph (local default)' + : 'oxigraph-server (default)' }`, ); { diff --git a/packages/cli/src/commands/lifecycle.ts b/packages/cli/src/commands/lifecycle.ts index 5dace2f9ba..c4c8723c68 100644 --- a/packages/cli/src/commands/lifecycle.ts +++ b/packages/cli/src/commands/lifecycle.ts @@ -290,7 +290,7 @@ program // remote store. Quad count = null is rare in practice (cached // every 30 s on the daemon side) so when it shows up the // operator should treat it as an alert, not a no-op. - const backend = s.storeBackend ?? 'oxigraph-worker'; + const backend = s.storeBackend ?? 'oxigraph-server'; if (s.storeUrl) { const quads = s.storeQuads == null ? 'UNREACHABLE' : `${s.storeQuads.toLocaleString()} quads`; console.log(` Store: ${backend} (${s.storeUrl}) — ${quads}`); diff --git a/packages/cli/src/commands/mcp.ts b/packages/cli/src/commands/mcp.ts index 9ff4c39d9c..d11ad64eaf 100644 --- a/packages/cli/src/commands/mcp.ts +++ b/packages/cli/src/commands/mcp.ts @@ -29,6 +29,7 @@ import { import { ApiClient } from '../api-client.js'; import { parsePositiveIntegerOption, parsePositiveMsOption } from '../cli-option-parsers.js'; import { promptStoreBackend, applyStoreFlagsToConfig } from '../store-wizard.js'; +import { storeFlagBackendList } from '../store-backends.js'; import { runConfiguredSourceWorker } from '../source-worker-runner.js'; import { batchEntityQuads } from '../batching.js'; import { @@ -157,7 +158,7 @@ mcpCmd .option('--yes', 'Auto-confirm per-client registrations (default false: prompt interactively in TTY mode; non-TTY auto-confirms — pass `--yes` in scripts for the safer scripted-environment posture)') .option( '--store ', - 'Triple-store backend (oxigraph | blazegraph | sparql-http). Validates the URL and persists the store block after setup.', + `Triple-store backend (${storeFlagBackendList(' | ')}). Validates the URL and persists the store block after setup.`, ) .option( '--store-url ', diff --git a/packages/cli/src/commands/openclaw.ts b/packages/cli/src/commands/openclaw.ts index 1d23846f7e..d0fa61b47c 100644 --- a/packages/cli/src/commands/openclaw.ts +++ b/packages/cli/src/commands/openclaw.ts @@ -32,6 +32,7 @@ import { import { ApiClient } from '../api-client.js'; import { parsePositiveIntegerOption, parsePositiveMsOption } from '../cli-option-parsers.js'; import { promptStoreBackend, applyStoreFlagsToConfig } from '../store-wizard.js'; +import { storeFlagBackendList } from '../store-backends.js'; import { runConfiguredSourceWorker } from '../source-worker-runner.js'; import { batchEntityQuads } from '../batching.js'; import { @@ -127,7 +128,7 @@ openclawCmd ) .option( '--store ', - 'Triple-store backend (oxigraph | blazegraph | sparql-http). Validates the URL via an ASK probe and persists the store block after setup completes.', + `Triple-store backend (${storeFlagBackendList(' | ')}). Validates the URL via an ASK probe and persists the store block after setup completes.`, ) .option( '--store-url ', diff --git a/packages/cli/src/config.ts b/packages/cli/src/config.ts index 6e4fa09152..dd2baa784a 100644 --- a/packages/cli/src/config.ts +++ b/packages/cli/src/config.ts @@ -19,6 +19,12 @@ import { STORAGE_ACK_TIMING_SAFETY_MARGIN_MS, type StorageAckTiming, } from '@origintrail-official/dkg-publisher'; +import { + getStoreBackendPolicy, + isExternalStoreBackend, + isRetiredStoreBackend, + configBackendList, +} from './store-backends.js'; /** * Per-step build timeouts (milliseconds) used by the git-based auto-update @@ -574,7 +580,7 @@ export interface DkgConfig { llm?: LlmConfig; /** Block explorer URL for TX links (default: derived from chainId). */ blockExplorerUrl?: string; - /** Triple store backend override (default: oxigraph-worker with file persistence). */ + /** Triple store backend override (default: daemon-managed oxigraph-server). */ store?: { backend: string; options?: Record; graphSetIndex?: boolean | GraphSetIndexConfig; changelog?: boolean }; /** * Intentional cap on how many persisted context-graph subscriptions a node @@ -1831,34 +1837,31 @@ export interface StoreConfigValidationError { export function validateStoreConfig(config: DkgConfig): StoreConfigValidationError[] { const errors: StoreConfigValidationError[] = []; const backend = config.store?.backend; - // Mirror of `isExternalBackend` from @origintrail-official/dkg-storage. - // Duplicated here to keep config.ts free of upward dependencies on the - // storage package (config.ts is leaf-imported by many other modules). - const isExternal = backend === 'blazegraph' || backend === 'sparql-http'; - if (!isExternal) return errors; + if (isRetiredStoreBackend(backend)) { + return [{ + field: 'store.backend', + message: + `${EXTERNAL_VALIDATION_PREFIX} "${backend}" is no longer supported. ` + + `Use one of: ${configBackendList()}.`, + }]; + } + if (!isExternalStoreBackend(backend)) return errors; const opts = (config.store?.options ?? {}) as Record; - - if (backend === 'blazegraph') { - if (typeof opts.url !== 'string' || !opts.url.trim()) { - errors.push({ - field: 'store.options.url', - message: - `${EXTERNAL_VALIDATION_PREFIX} is "blazegraph" but ` + - `store.options.url is missing. Set it to the SPARQL endpoint URL ` + - `(e.g. http://127.0.0.1:9999/bigdata/namespace/mynode/sparql) or ` + - `switch backend to oxigraph-worker.`, - }); - } - } else if (backend === 'sparql-http') { - if (typeof opts.queryEndpoint !== 'string' || !opts.queryEndpoint.trim()) { - errors.push({ - field: 'store.options.queryEndpoint', - message: - `${EXTERNAL_VALIDATION_PREFIX} is "sparql-http" but ` + - `store.options.queryEndpoint is missing. Set it to the SPARQL query URL.`, - }); - } + const policy = getStoreBackendPolicy(backend); + if (!policy || policy.kind !== 'external') { + throw new Error(`Missing external-store policy for "${backend}"`); + } + const queryOption = policy.queryEndpointOption; + const queryEndpoint = opts[queryOption]; + if (typeof queryEndpoint !== 'string' || !queryEndpoint.trim()) { + errors.push({ + field: `store.options.${queryOption}`, + message: + `${EXTERNAL_VALIDATION_PREFIX} is "${backend}" but ` + + `store.options.${queryOption} is missing. Set it to the SPARQL query endpoint URL ` + + `or switch backend to oxigraph-server.`, + }); } if (config.largeLiteralStorage?.enabled === true) { diff --git a/packages/cli/src/daemon/chain-reset-wipe.ts b/packages/cli/src/daemon/chain-reset-wipe.ts index 2bb6df3a2c..6d56c8bde4 100644 --- a/packages/cli/src/daemon/chain-reset-wipe.ts +++ b/packages/cli/src/daemon/chain-reset-wipe.ts @@ -64,8 +64,6 @@ */ import { existsSync, - readFileSync, - writeFileSync, readdirSync, rmSync, renameSync, @@ -74,32 +72,14 @@ import { } from 'node:fs'; import { join } from 'node:path'; import { isExternalBackend, getSparqlEndpoint, CHANGELOG_GRAPH } from '@origintrail-official/dkg-storage'; - -const STATE_FILE = '.network-state.json'; - -interface PersistedNetworkState { - /** Last chainResetMarker value the daemon booted on. */ - chainResetMarker: string | null; - /** - * Last triple-store backend the daemon booted on. Used by - * `detectBackendSwitch` to warn loudly when an operator hand-edits - * `config.store.backend` between boots — the new backend is fresh - * and empty, so silently booting would mean stale SWM/VM data is - * inaccessible. `null` on legacy state files (pre-RFC 120) and on - * first boot. (RFC 120 review point #6.) - */ - lastBackend?: string | null; - /** - * Last resolved network the daemon booted on (the `networkConfig` overlay - * name, e.g. `mainnet-gnosis`/`testnet`). Used by `detectNetworkSwitch` to - * abort boot when an operator repoints `config.networkConfig` at a different - * network on an existing data dir — the store holds the old network's - * chain-derived state (KC ids, merkle roots), which is meaningless on the - * new chain. `null`/absent on legacy state files and on first boot. - */ - lastNetworkConfig?: string | null; - savedAt: number; -} +import { + readPersistedDaemonState, + readPersistedNetworkConfig, + readPersistedStoreBackend, + writePersistedChainResetMarker, + writePersistedNetworkConfig, + writePersistedStoreBackend, +} from './daemon-state.js'; /** * Subset of `DkgConfig['store']` used by the wipe step to talk to an @@ -229,69 +209,6 @@ export function skipChainResetWipe(env: NodeJS.ProcessEnv = process.env): boolea return env.DKG_SKIP_CHAIN_RESET_WIPE === '1'; } -function loadState(dataDir: string): PersistedNetworkState | null { - try { - const raw = readFileSync(join(dataDir, STATE_FILE), 'utf8'); - const obj = JSON.parse(raw) as PersistedNetworkState; - if (typeof obj?.chainResetMarker !== 'string' && obj?.chainResetMarker !== null) return null; - return obj; - } catch { - return null; - } -} - -function saveState(dataDir: string, marker: string | null): void { - // Preserve any sibling fields (lastBackend) that `detectBackendSwitch` - // may have written. Otherwise a chain-reset wipe would clobber a - // freshly-recorded backend tag and the next boot would re-warn. - const existing = loadState(dataDir) ?? { chainResetMarker: null, savedAt: 0 }; - writeFileSync( - join(dataDir, STATE_FILE), - JSON.stringify( - { - ...existing, - chainResetMarker: marker, - savedAt: Date.now(), - } satisfies PersistedNetworkState, - null, - 2, - ), - ); -} - -function saveBackendTag(dataDir: string, backend: string): void { - const existing = loadState(dataDir) ?? { chainResetMarker: null, savedAt: 0 }; - writeFileSync( - join(dataDir, STATE_FILE), - JSON.stringify( - { - ...existing, - lastBackend: backend, - savedAt: Date.now(), - } satisfies PersistedNetworkState, - null, - 2, - ), - ); -} - -function saveNetworkTag(dataDir: string, networkConfig: string): void { - // Preserve sibling fields (chainResetMarker, lastBackend) like saveBackendTag. - const existing = loadState(dataDir) ?? { chainResetMarker: null, savedAt: 0 }; - writeFileSync( - join(dataDir, STATE_FILE), - JSON.stringify( - { - ...existing, - lastNetworkConfig: networkConfig, - savedAt: Date.now(), - } satisfies PersistedNetworkState, - null, - 2, - ), - ); -} - /** * Wipe the V10 data sitting in an external SPARQL endpoint. Runs after * the local file wipe so we don't strand the operator with a wiped FS @@ -582,7 +499,7 @@ export async function chainResetWipe( return { wiped: false, skipped: false, prevMarker: null, removedFiles: [], backedUpFiles: [], failedFiles: [] }; } - const prev = loadState(opts.dataDir); + const prev = readPersistedDaemonState(opts.dataDir); const prevMarker = prev?.chainResetMarker ?? null; if (prevMarker === opts.currentMarker) { @@ -664,7 +581,7 @@ export async function chainResetWipe( if (failedFiles.length === 0) { try { - saveState(opts.dataDir, opts.currentMarker); + writePersistedChainResetMarker(opts.dataDir, opts.currentMarker); markerPersisted = true; } catch (err) { log( @@ -718,7 +635,7 @@ export interface BackendSwitchDetectOptions { /** * Backend name from the current config. Pass the effective value * including the default — e.g. when `config.store?.backend` is - * undefined, callers should pass `'oxigraph-worker'` so the check + * undefined, callers should pass `'oxigraph-server'` so the check * is symmetric across "no store block" ↔ "explicit store block". */ currentBackend: string; @@ -750,21 +667,17 @@ export function detectBackendSwitch( opts: BackendSwitchDetectOptions, ): BackendSwitchDetectResult { const log = opts.log ?? (() => {}); - const prev = loadState(opts.dataDir); - const previous = - typeof prev?.lastBackend === 'string' && prev.lastBackend.length > 0 - ? prev.lastBackend - : null; + const previous = readPersistedStoreBackend(opts.dataDir); // First boot or legacy state file: silently record and move on. We - // explicitly do NOT treat null-previous as a "switch from - // oxigraph-worker"; that would re-warn every operator who upgrades + // explicitly do NOT treat null-previous as a "switch from the old + // implicit backend"; that would re-warn every operator who upgrades // into this release without ever having touched their store // configuration. Only operator-visible config changes between two // recorded backends count as a switch. if (previous === null) { try { - saveBackendTag(opts.dataDir, opts.currentBackend); + writePersistedStoreBackend(opts.dataDir, opts.currentBackend); } catch { // Non-fatal: if we can't write the tag now, we'll try again next // boot. The downside is one missed early-warning window. @@ -801,7 +714,7 @@ export function detectBackendSwitch( log(``); log(`DKG_ACCEPT_STORE_RESET=1 set — proceeding with the new backend.`); try { - saveBackendTag(opts.dataDir, opts.currentBackend); + writePersistedStoreBackend(opts.dataDir, opts.currentBackend); } catch (err) { log(`WARN: failed to persist new backend tag: ${(err as Error).message}. Will re-warn on next boot.`); } @@ -851,11 +764,7 @@ export function detectNetworkSwitch( opts: NetworkSwitchDetectOptions, ): NetworkSwitchDetectResult { const log = opts.log ?? (() => {}); - const prev = loadState(opts.dataDir); - const previous = - typeof prev?.lastNetworkConfig === 'string' && prev.lastNetworkConfig.length > 0 - ? prev.lastNetworkConfig - : null; + const previous = readPersistedNetworkConfig(opts.dataDir); // First boot or legacy state file: silently record and move on. We do NOT // treat null-previous as a switch — that would abort every operator who @@ -866,7 +775,7 @@ export function detectNetworkSwitch( // are caught normally. if (previous === null) { try { - saveNetworkTag(opts.dataDir, opts.currentNetworkConfig); + writePersistedNetworkConfig(opts.dataDir, opts.currentNetworkConfig); } catch { // Non-fatal: retry the tag write next boot. } @@ -906,7 +815,7 @@ export function detectNetworkSwitch( log(``); log(`DKG_ACCEPT_NETWORK_SWITCH=1 set — proceeding on the new network.`); try { - saveNetworkTag(opts.dataDir, opts.currentNetworkConfig); + writePersistedNetworkConfig(opts.dataDir, opts.currentNetworkConfig); } catch (err) { log(`WARN: failed to persist new network tag: ${(err as Error).message}. Will re-warn on next boot.`); } diff --git a/packages/cli/src/daemon/daemon-state.ts b/packages/cli/src/daemon/daemon-state.ts new file mode 100644 index 0000000000..66d546821e --- /dev/null +++ b/packages/cli/src/daemon/daemon-state.ts @@ -0,0 +1,76 @@ +import { readFileSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +export const DAEMON_STATE_FILE = '.network-state.json'; + +/** Persisted boot state shared by reset and configuration-switch guards. */ +export interface PersistedDaemonState { + chainResetMarker: string | null; + lastBackend?: string | null; + lastNetworkConfig?: string | null; + savedAt: number; +} + +export function readPersistedDaemonState(dataDir: string): PersistedDaemonState | null { + try { + const raw = readFileSync(join(dataDir, DAEMON_STATE_FILE), 'utf8'); + const state = JSON.parse(raw) as PersistedDaemonState; + if ( + typeof state?.chainResetMarker !== 'string' + && state?.chainResetMarker !== null + ) { + return null; + } + return state; + } catch { + return null; + } +} + +function updatePersistedDaemonState( + dataDir: string, + patch: Partial>, +): void { + const existing = readPersistedDaemonState(dataDir) ?? { + chainResetMarker: null, + savedAt: 0, + }; + writeFileSync( + join(dataDir, DAEMON_STATE_FILE), + JSON.stringify( + { + ...existing, + ...patch, + savedAt: Date.now(), + } satisfies PersistedDaemonState, + null, + 2, + ), + ); +} + +export function readPersistedStoreBackend(dataDir: string): string | null { + const state = readPersistedDaemonState(dataDir); + return typeof state?.lastBackend === 'string' && state.lastBackend.length > 0 + ? state.lastBackend + : null; +} + +export function readPersistedNetworkConfig(dataDir: string): string | null { + const state = readPersistedDaemonState(dataDir); + return typeof state?.lastNetworkConfig === 'string' && state.lastNetworkConfig.length > 0 + ? state.lastNetworkConfig + : null; +} + +export function writePersistedChainResetMarker(dataDir: string, marker: string | null): void { + updatePersistedDaemonState(dataDir, { chainResetMarker: marker }); +} + +export function writePersistedStoreBackend(dataDir: string, backend: string): void { + updatePersistedDaemonState(dataDir, { lastBackend: backend }); +} + +export function writePersistedNetworkConfig(dataDir: string, networkConfig: string): void { + updatePersistedDaemonState(dataDir, { lastNetworkConfig: networkConfig }); +} diff --git a/packages/cli/src/daemon/handle-request.ts b/packages/cli/src/daemon/handle-request.ts index 4297ac48c3..918080a9f8 100644 --- a/packages/cli/src/daemon/handle-request.ts +++ b/packages/cli/src/daemon/handle-request.ts @@ -311,7 +311,12 @@ import { reverseLocalAgentSetupForUi, refreshLocalAgentIntegrationFromUi, } from './local-agents.js'; -import type { MemoryGraphChangedEvent, NotificationSseEvent, RequestContext } from './routes/context.js'; +import { + createRequestStoreContext, + type MemoryGraphChangedEvent, + type NotificationSseEvent, + type RequestContext, +} from './routes/context.js'; import { handleStatusRoutes } from './routes/status.js'; import { handleAgentChatRoutes } from './routes/agent-chat.js'; import { handleOpenclawRoutes } from './routes/openclaw.js'; @@ -330,6 +335,7 @@ import { handleOperationalWalletRoutes } from './routes/operational-wallets.js'; import { handleNotificationRoutes } from './routes/notifications.js'; import { handlePluginRoutes } from './routes/plugins.js'; import type { RoutePlugin } from './plugin-api.js'; +import type { StoreRuntimeContext } from './store-runtime.js'; export async function handleRequest( @@ -338,7 +344,7 @@ export async function handleRequest( agent: DKGAgent, publisherControl: ReturnType, publisherRuntime: PublisherRuntime | null, - config: DkgConfig, + storeRuntime: StoreRuntimeContext, startedAt: number, dashDb: DashboardDB, opWallets: import("@origintrail-official/dkg-agent").OpWalletsConfig, @@ -383,7 +389,7 @@ export async function handleRequest( publisherControl, publisherRuntime, publisherAvailability, - config, + ...createRequestStoreContext(storeRuntime), startedAt, dashDb, opWallets, diff --git a/packages/cli/src/daemon/lifecycle.ts b/packages/cli/src/daemon/lifecycle.ts index 6aad669619..1fa6438cac 100644 --- a/packages/cli/src/daemon/lifecycle.ts +++ b/packages/cli/src/daemon/lifecycle.ts @@ -40,7 +40,7 @@ import { import { execSync, exec, execFile } from "node:child_process"; import { promisify } from "node:util"; import { join, dirname, resolve } from 'node:path'; -import { existsSync, readdirSync, readFileSync, openSync, closeSync, writeFileSync as fsWriteFileSync, unlinkSync } from 'node:fs'; +import { readdirSync, readFileSync, openSync, closeSync, writeFileSync as fsWriteFileSync, unlinkSync } from 'node:fs'; // Namespace import: our Phase-8 install-context builder (~line 290) calls // `osModule.homedir()`, and the later agent-identity probe (~line 6851) // uses `osModule.hostname()` + `osModule.userInfo()`. v10-rc's new @@ -306,6 +306,10 @@ import { } from './store-health-check.js'; import { startManagedOxigraph } from './oxigraph-managed.js'; import type { OxigraphServerHandle } from './oxigraph-server.js'; +import { + resolveDaemonStoreBootPlan, + resolveDaemonStoreRuntime, +} from './store-runtime.js'; import { resetNatStatus, startNatStatusWatcher } from './nat-status.js'; import { OPENCLAW_UI_CONNECT_TIMEOUT_MS, @@ -1127,6 +1131,25 @@ export async function runDaemonInner( ...resolveNetworkDefaultContextGraphs(network), ]), ]; + const acceptStoreReset = process.env.DKG_ACCEPT_STORE_RESET === '1'; + const storeDecision = resolveDaemonStoreBootPlan({ + config, + dataDir: dkgDir(), + acceptStoreReset, + }); + + if (storeDecision.kind === 'invalid-config') { + exitOnStoreConfigErrors(storeDecision.operatorConfig, log); + throw new Error('Invalid store config validation unexpectedly returned'); + } + if (storeDecision.kind === 'blocked-legacy-cutover') { + log(storeDecision.message); + process.exit(1); + } + const storeBoot = storeDecision; + if (storeBoot.notice) { + log(storeBoot.notice); + } // Auto-wipe per-node chain-state derived files (oxigraph store, publish // journal, random-sampling WAL) when the maintainer bumps @@ -1143,8 +1166,8 @@ export async function runDaemonInner( // silently would look like data loss to the operator. const backendSwitch = detectBackendSwitch({ dataDir: dkgDir(), - currentBackend: config.store?.backend ?? 'oxigraph-worker', - acceptStoreReset: process.env.DKG_ACCEPT_STORE_RESET === '1', + currentBackend: storeBoot.effectiveStore.backend, + acceptStoreReset, log, }); if (backendSwitch.aborted) { @@ -1180,7 +1203,7 @@ export async function runDaemonInner( let managedOxigraph: OxigraphServerHandle | null = null; let managed: Awaited> = null; try { - managed = await startManagedOxigraph({ config, dataDir: dkgDir(), log }); + managed = await startManagedOxigraph({ config: storeBoot.effectiveConfig, dataDir: dkgDir(), log }); if (managed) { managedOxigraph = managed.handle; // Every remaining fatal boot path (config validation, store health @@ -1193,7 +1216,7 @@ export async function runDaemonInner( } catch (err) { log( `[STORE] failed to start managed Oxigraph server: ${(err as Error).message}\n` + - `Fix the cause, or switch \`store.backend\` to oxigraph-worker (embedded) or ` + + `Fix the cause, or switch \`store.backend\` to ` + `sparql-http (operator-managed endpoint) in ~/.dkg/config.json.`, ); process.exit(1); @@ -1211,22 +1234,13 @@ export async function runDaemonInner( // For the directory-backed blob/snapshot stores we use the managed // defaults (the rewritten sparql-http backend has no `options.path` to // infer a directory from, unlike the local Oxigraph backend). - const runtimeStore = managed?.storeConfig ?? config.store; - const runtimeLargeLiteralStorage = - managed?.largeLiteralStorage ?? config.largeLiteralStorage; - const runtimeSnapshotStorage = - managed?.sharedMemoryPublicSnapshotStorage ?? config.sharedMemoryPublicSnapshotStorage; - // Config view used only for the boot-time store validation/health steps - // below: same as `config` but with the runtime store/blob/snapshot values - // swapped in, so a managed config validates against what actually runs. - const runtimeStoreConfig: DkgConfig = managed - ? { - ...config, - store: runtimeStore, - largeLiteralStorage: runtimeLargeLiteralStorage, - sharedMemoryPublicSnapshotStorage: runtimeSnapshotStorage, - } - : config; + const storeRuntime = resolveDaemonStoreRuntime(storeBoot, managed); + const { + runtimeStore, + runtimeConfig: runtimeStoreConfig, + runtimeLargeLiteralStorage, + runtimeSnapshotStorage, + } = storeRuntime; // Refuse to start on invalid external-backend config (missing URL, // missing blob/snapshot directory). This fires before the health @@ -1979,7 +1993,7 @@ export async function runDaemonInner( try { const runtime = await startPublisherRuntimeIfEnabled({ dataDir: dkgDir(), - config, + config: runtimeStoreConfig, store: agent.store, keypair: agent.wallet.keypair, chainBase: publisherChainBase, @@ -3360,7 +3374,7 @@ export async function runDaemonInner( agent, publisherControl, publisherRuntime, - config, + storeRuntime, startedAt, dashDb, opWallets, diff --git a/packages/cli/src/daemon/oxigraph-managed.ts b/packages/cli/src/daemon/oxigraph-managed.ts index 1dd6cd6b5f..948505a601 100644 --- a/packages/cli/src/daemon/oxigraph-managed.ts +++ b/packages/cli/src/daemon/oxigraph-managed.ts @@ -23,6 +23,7 @@ * matching the Blazegraph-Docker provisioner's contract. */ import { join } from 'node:path'; +import { MANAGED_DAEMON_STORE_BACKEND } from '../store-backends.js'; import { ensureOxigraphBinary } from './oxigraph-binary.js'; import { startOxigraphServer, @@ -35,7 +36,7 @@ import { } from './oxigraph-launch-strategy.js'; /** Config value that opts a node into the daemon-managed local server. */ -export const MANAGED_OXIGRAPH_BACKEND = 'oxigraph-server'; +export { MANAGED_DAEMON_STORE_BACKEND as MANAGED_OXIGRAPH_BACKEND }; /** Default loopback bind port. Override via `store.options.port`. */ export const DEFAULT_OXIGRAPH_PORT = 7878; @@ -144,7 +145,7 @@ export function planManagedOxigraph( config: ConfigLike, dataDir: string, ): ManagedOxigraphPlan | null { - if (config.store?.backend !== MANAGED_OXIGRAPH_BACKEND) return null; + if (config.store?.backend !== MANAGED_DAEMON_STORE_BACKEND) return null; const options = config.store.options ?? {}; const port = resolveManagedOxigraphPort(options); diff --git a/packages/cli/src/daemon/routes/context.ts b/packages/cli/src/daemon/routes/context.ts index 8b37447315..c29268c4b4 100644 --- a/packages/cli/src/daemon/routes/context.ts +++ b/packages/cli/src/daemon/routes/context.ts @@ -23,6 +23,7 @@ import type { VectorStore, EmbeddingProvider } from '../../vector-store.js'; import type { CatchupTracker } from '../types.js'; import type { RoutePlugin } from '../plugin-api.js'; import type { AdmissionStatsView } from '../http-utils.js'; +import type { StoreRuntimeContext } from '../store-runtime.js'; export type MemoryGraphLayer = 'wm' | 'swm' | 'vm'; @@ -54,7 +55,29 @@ export interface NotificationSseEvent { type: string; } -export interface RequestContext { +/** + * Store views exposed to routes. The operator config is intentionally the only + * config object in this shape, so a direct route harness cannot provide a + * second, contradictory operator config through a nested store context. + */ +export interface RequestStoreContext { + /** Operator config exactly as loaded from disk / CLI. */ + config: DkgConfig; + /** Daemon-facing backend after defaults and acknowledged migrations. */ + effectiveStore: StoreRuntimeContext['effectiveStore']; + /** Constructible live adapter config after managed-store materialization. */ + runtimeStore: StoreRuntimeContext['runtimeStore']; +} + +export function createRequestStoreContext(storeRuntime: StoreRuntimeContext): RequestStoreContext { + return { + config: storeRuntime.operatorConfig, + effectiveStore: storeRuntime.effectiveStore, + runtimeStore: storeRuntime.runtimeStore, + }; +} + +export interface RequestContext extends RequestStoreContext { req: IncomingMessage; res: ServerResponse; agent: DKGAgent; @@ -62,7 +85,6 @@ export interface RequestContext { publisherRuntime: PublisherRuntime | null; /** Lifecycle-owned publisher state; optional for direct route embeddings/tests. */ publisherAvailability?: AsyncPublisherAvailability; - config: DkgConfig; startedAt: number; dashDb: DashboardDB; opWallets: OpWalletsConfig; diff --git a/packages/cli/src/daemon/routes/status.ts b/packages/cli/src/daemon/routes/status.ts index 1ddf8eade3..aa052f979e 100644 --- a/packages/cli/src/daemon/routes/status.ts +++ b/packages/cli/src/daemon/routes/status.ts @@ -57,8 +57,10 @@ const execAsync = promisify(exec); const execFileAsync = promisify(execFile); import { enrichEvmError, MockChainAdapter, resolveRpcUrls, getRpcFailoverStats } from '@origintrail-official/dkg-chain'; import { DKGAgent, loadOpWallets } from '@origintrail-official/dkg-agent'; -import { isExternalBackend } from '@origintrail-official/dkg-storage'; -import { resolveManagedOxigraphPort } from '../oxigraph-managed.js'; +import { + isExternalStoreBackend as isExternalBackend, + isManagedLocalBackend, +} from '../../store-backends.js'; import { computeNetworkId, createOperationContext, DKGEvent, Logger, PayloadTooLargeError, GET_VIEWS, TrustLevel, validateSubGraphName, validateAssertionName, validateContextGraphId, isSafeIri, assertSafeIri, sparqlIri, contextGraphSharedMemoryUri, contextGraphAssertionUri, contextGraphMetaUri } from '@origintrail-official/dkg-core'; import { findReservedSubjectPrefix, isSkolemizedUri } from '@origintrail-official/dkg-publisher'; import { @@ -420,6 +422,10 @@ export function invalidateExternalStoreQuadsCache(): void { storeQuadsInflight = null; } +export function storeBackendHasStatusHealth(backend: string | undefined): boolean { + return isExternalBackend(backend) || isManagedLocalBackend(backend); +} + async function getCachedExternalStoreQuads( agent: DKGAgent, now: number, @@ -498,6 +504,8 @@ export async function handleStatusRoutes(ctx: RequestContext): Promise { agent, publisherControl, config, + effectiveStore, + runtimeStore, startedAt, dashDb, opWallets, @@ -651,6 +659,7 @@ export async function handleStatusRoutes(ctx: RequestContext): Promise { // sentinels when build-info.json is absent (monorepo / dev), // so consumers can branch reliably. const buildInfo = loadBuildInfo(); + const runtimeStoreOptions = (runtimeStore.options ?? {}) as Record; return jsonResponse(res, 200, { name: config.name, version: nodeVersion, @@ -666,36 +675,25 @@ export async function handleStatusRoutes(ctx: RequestContext): Promise { networkConfig: resolveNetworkConfigName(config), networkId, networkName: network?.networkName ?? null, - storeBackend: config.store?.backend ?? "oxigraph-worker", + storeBackend: effectiveStore.backend, // External backend visibility (RFC 120 / plan PR 1 item 3). For // local backends both fields stay null so the response shape is // stable across deployments. - storeUrl: isExternalBackend(config.store?.backend) + storeUrl: isExternalBackend(runtimeStore.backend) ? (() => { - const opts = (config.store?.options ?? {}) as Record; - const url = typeof opts.url === 'string' ? opts.url - : typeof opts.queryEndpoint === 'string' ? opts.queryEndpoint + const url = typeof runtimeStoreOptions.url === 'string' ? runtimeStoreOptions.url + : typeof runtimeStoreOptions.queryEndpoint === 'string' ? runtimeStoreOptions.queryEndpoint : null; return url; })() - : config.store?.backend === 'oxigraph-server' - // Managed local server: report its loopback endpoint so `dkg status` - // renders the external-store health path (storeQuads/unreachable) - // instead of printing it like a quad-less local store. - ? (() => { - const opts = (config.store?.options ?? {}) as Record; - const port = resolveManagedOxigraphPort(opts); - return `http://127.0.0.1:${port}/query`; - })() - : null, - // A managed `oxigraph-server` keeps `config.store.backend` as - // "oxigraph-server" (so it persists/labels correctly), but its quad - // count is still worth surfacing — it's the only store-health signal - // for that backend (getStoreBytes is null, there's no store.nq), and a - // failed query here is how operators see the managed server is down - // (e.g. after a failed revive) instead of it always looking healthy. + : null, + // `storeBackend` describes effective daemon policy while URL and health + // come from the live constructible adapter. This matters for both the + // implicit managed default and acknowledged oxigraph-worker cutovers: + // operator config may be absent/retired, effective is oxigraph-server, + // and runtime is its materialized loopback sparql-http adapter. storeQuads: - isExternalBackend(config.store?.backend) || config.store?.backend === 'oxigraph-server' + (storeBackendHasStatusHealth(effectiveStore.backend) || isExternalBackend(runtimeStore.backend)) ? await getCachedExternalStoreQuads(agent, Date.now()) : null, uptimeMs: Date.now() - startedAt, diff --git a/packages/cli/src/daemon/store-runtime.ts b/packages/cli/src/daemon/store-runtime.ts new file mode 100644 index 0000000000..6f43d6dc15 --- /dev/null +++ b/packages/cli/src/daemon/store-runtime.ts @@ -0,0 +1,174 @@ +import { existsSync } from 'node:fs'; +import { join } from 'node:path'; +import { + DEFAULT_DAEMON_STORE_BACKEND, + isManagedLocalBackend, + isRetiredStoreBackend, + requireStorageAdapterBackend, + type StorageAdapterBackend, +} from '../store-backends.js'; +import type { DkgConfig } from '../config.js'; +import { readPersistedStoreBackend } from './daemon-state.js'; +import type { ManagedOxigraphResult } from './oxigraph-managed.js'; + +type StoreConfig = NonNullable; +export type RuntimeStoreConfig = Omit & { + backend: StorageAdapterBackend; +}; + +interface DaemonStoreOperatorContext { + /** Operator-facing config exactly as loaded from disk / CLI. */ + operatorConfig: DkgConfig; +} + +export interface InvalidDaemonStoreConfig extends DaemonStoreOperatorContext { + kind: 'invalid-config'; +} + +export interface BlockedLegacyStoreCutover extends DaemonStoreOperatorContext { + kind: 'blocked-legacy-cutover'; + message: string; +} + +export interface DaemonStoreBootPlan extends DaemonStoreOperatorContext { + kind: 'bootable'; + /** Config with the implicit daemon default materialized for boot steps. */ + effectiveConfig: DkgConfig; + /** Store backend used for backend-switch detection and managed startup. */ + effectiveStore: StoreConfig; + /** Non-fatal startup notice, e.g. acknowledged legacy default cutover. */ + notice?: string; +} + +export type DaemonStoreBootDecision = + | InvalidDaemonStoreConfig + | BlockedLegacyStoreCutover + | DaemonStoreBootPlan; + +export interface DaemonStoreRuntimePlan extends DaemonStoreBootPlan { + /** Store config consumed by validation, health probes, wipe, and the agent. */ + runtimeStore: RuntimeStoreConfig; + /** Config view with runtime store/blob/snapshot values swapped in. */ + runtimeConfig: DkgConfig; + runtimeLargeLiteralStorage: DkgConfig['largeLiteralStorage']; + runtimeSnapshotStorage: DkgConfig['sharedMemoryPublicSnapshotStorage']; +} + +/** Explicit store views threaded into request routing after startup. */ +export interface StoreRuntimeContext { + /** Persisted/operator intent. Routes that save config must use this view. */ + operatorConfig: DkgConfig; + /** Daemon-facing backend after defaults and acknowledged migrations. */ + effectiveStore: StoreConfig; + /** Constructible live adapter config after managed-store materialization. */ + runtimeStore: RuntimeStoreConfig; +} + +export function resolveEffectiveDaemonStore(config: Pick): StoreConfig { + return config.store ?? { backend: DEFAULT_DAEMON_STORE_BACKEND, options: {} }; +} + +export function resolveDaemonStoreBootPlan(opts: { + config: DkgConfig; + dataDir: string; + acceptStoreReset: boolean; +}): DaemonStoreBootDecision { + const { config, dataDir, acceptStoreReset } = opts; + const legacyStorePath = join(dataDir, 'store.nq'); + const legacyStoreExists = existsSync(legacyStorePath); + const retiredStoreConfigured = isRetiredStoreBackend(config.store?.backend); + const configuredForManagedServer = !config.store || isManagedLocalBackend(config.store.backend); + const previousBackend = readPersistedStoreBackend(dataDir); + const legacyCutoverAlreadyRecorded = isManagedLocalBackend(previousBackend); + const legacyCutoverRequired = legacyStoreExists + && (configuredForManagedServer || retiredStoreConfigured) + && !legacyCutoverAlreadyRecorded; + const migrateAcknowledgedRetiredStore = retiredStoreConfigured + && legacyCutoverRequired + && acceptStoreReset; + const effectiveStore = migrateAcknowledgedRetiredStore + ? resolveEffectiveDaemonStore({}) + : resolveEffectiveDaemonStore(config); + const effectiveConfig = config.store && !migrateAcknowledgedRetiredStore + ? config + : { ...config, store: effectiveStore }; + + // A retired worker config with no legacy data has no migration decision to + // make. Keep that state separate so callers cannot accidentally continue to + // managed startup with an invalid operator config. + if (retiredStoreConfigured && !legacyCutoverRequired) { + return { kind: 'invalid-config', operatorConfig: config }; + } + + if (legacyCutoverRequired && !acceptStoreReset) { + const legacySource = retiredStoreConfigured + ? `${config.store?.backend} backend` + : config.store + ? 'worker-backed store' + : 'implicit worker default'; + return { + kind: 'blocked-legacy-cutover', + operatorConfig: config, + message: + `[STORE] oxigraph-worker support has been removed, but this node has a legacy ` + + `store.nq from the old ${legacySource}.\n` + + `Set store.backend to "oxigraph-server" (or an external SPARQL backend) and ` + + `restart with DKG_ACCEPT_STORE_RESET=1 to acknowledge the fresh-store cutover. ` + + `The legacy store.nq file is left untouched for manual backup or migration.`, + }; + } + + let notice: string | undefined; + if (legacyCutoverRequired && acceptStoreReset) { + notice = retiredStoreConfigured + ? `[STORE] explicit ${config.store?.backend} is retired; using oxigraph-server after reset acknowledgement. Legacy store.nq is left untouched.` + : '[STORE] using oxigraph-server after reset acknowledgement. Legacy store.nq is left untouched.'; + } else if (!config.store && acceptStoreReset) { + notice = + '[STORE] no store block found; using oxigraph-server. Legacy store.nq, if present, is left untouched.'; + } + + return { + kind: 'bootable', + operatorConfig: config, + effectiveConfig, + effectiveStore, + ...(notice ? { notice } : {}), + }; +} + +export function resolveDaemonStoreRuntime( + bootPlan: DaemonStoreBootPlan, + managed: ManagedOxigraphResult | null, +): DaemonStoreRuntimePlan { + if (isManagedLocalBackend(bootPlan.effectiveStore.backend) && !managed) { + throw new Error( + `Managed daemon store "${bootPlan.effectiveStore.backend}" was not materialized to a storage adapter`, + ); + } + const candidateStore = managed?.storeConfig ?? bootPlan.effectiveStore; + const runtimeStore: RuntimeStoreConfig = { + ...candidateStore, + backend: requireStorageAdapterBackend(candidateStore.backend), + }; + const runtimeLargeLiteralStorage = + managed?.largeLiteralStorage ?? bootPlan.operatorConfig.largeLiteralStorage; + const runtimeSnapshotStorage = + managed?.sharedMemoryPublicSnapshotStorage ?? bootPlan.operatorConfig.sharedMemoryPublicSnapshotStorage; + const runtimeConfig: DkgConfig = managed + ? { + ...bootPlan.effectiveConfig, + store: runtimeStore, + largeLiteralStorage: runtimeLargeLiteralStorage, + sharedMemoryPublicSnapshotStorage: runtimeSnapshotStorage, + } + : bootPlan.effectiveConfig; + + return { + ...bootPlan, + runtimeStore, + runtimeConfig, + runtimeLargeLiteralStorage, + runtimeSnapshotStorage, + }; +} diff --git a/packages/cli/src/publisher-runner.ts b/packages/cli/src/publisher-runner.ts index adac992a45..0d07d43c27 100644 --- a/packages/cli/src/publisher-runner.ts +++ b/packages/cli/src/publisher-runner.ts @@ -6,6 +6,7 @@ import { ACKCollector, AsyncLiftRunner, DKGPublisher, FileWorkspacePublicSnapsho import { createTripleStore, type TripleStore } from '@origintrail-official/dkg-storage'; import { loadNetworkConfig, resolveReadyChainConfig, type DkgConfig } from './config.js'; import { loadPublisherWallets } from './publisher-wallets.js'; +import { isManagedLocalBackend, isRetiredStoreBackend } from './store-backends.js'; export type { ACKTransportFactory } from '@origintrail-official/dkg-publisher'; @@ -588,6 +589,18 @@ function createChainRecoveryResolver( async function createPublisherStore(dataDir: string, config: DkgConfig): Promise { if (config.store) { + if (isManagedLocalBackend(config.store.backend)) { + throw new Error( + `Publisher commands for daemon-managed store "${config.store.backend}" require a running DKG daemon. ` + + 'Start the daemon and retry; daemon-down direct inspection cannot safely materialize the managed store.', + ); + } + if (isRetiredStoreBackend(config.store.backend)) { + throw new Error( + `Publisher commands cannot open retired store backend "${config.store.backend}" directly. ` + + 'Start the daemon to complete the acknowledged store migration, then retry.', + ); + } const storeConfig = config.store as any; return await createTripleStore({ ...storeConfig, @@ -606,7 +619,7 @@ async function createPublisherStore(dataDir: string, config: DkgConfig): Promise } return await createTripleStore({ - backend: 'oxigraph-worker', + backend: 'oxigraph-persistent', options: { path: join(dataDir, 'store.nq') }, largeLiteralStorage: defaultLargeLiteralStorage(dataDir, config), }); @@ -633,7 +646,6 @@ export function createPublicSnapshotStore( function isLocalOxigraphStoreConfig(storeConfig: { backend?: unknown }): boolean { return storeConfig.backend === 'oxigraph' - || storeConfig.backend === 'oxigraph-worker' || storeConfig.backend === 'oxigraph-persistent'; } diff --git a/packages/cli/src/store-backends.ts b/packages/cli/src/store-backends.ts new file mode 100644 index 0000000000..87b78c8305 --- /dev/null +++ b/packages/cli/src/store-backends.ts @@ -0,0 +1,226 @@ +import { + STORAGE_ADAPTERS, + isExternalBackend, + isStorageAdapterBackend, + type StorageAdapterBackend, +} from '@origintrail-official/dkg-storage'; + +/** + * Operator-facing daemon policy composed on top of storage-owned adapter facts. + * + * This is the sole owner of daemon defaults, retired config names, migration + * classification, menu visibility, and labels. Adapter endpoint/path metadata + * is spread from `STORAGE_ADAPTERS` rather than duplicated here. + */ +export const STORE_BACKENDS = { + 'oxigraph-server': { + kind: 'managed-local', + adapter: false, + retired: false, + default: true, + wizard: true, + storeFlag: true, + label: 'oxigraph-server (managed local server — recommended)', + }, + oxigraph: { + ...STORAGE_ADAPTERS.oxigraph, + adapter: true, + retired: false, + default: false, + wizard: true, + storeFlag: true, + label: 'oxigraph (embedded in-memory store — development only)', + }, + 'oxigraph-persistent': { + ...STORAGE_ADAPTERS['oxigraph-persistent'], + adapter: true, + retired: false, + default: false, + wizard: false, + storeFlag: false, + }, + blazegraph: { + ...STORAGE_ADAPTERS.blazegraph, + adapter: true, + retired: false, + default: false, + wizard: true, + storeFlag: true, + label: 'blazegraph (external SPARQL endpoint)', + }, + 'sparql-http': { + ...STORAGE_ADAPTERS['sparql-http'], + adapter: true, + retired: false, + default: false, + wizard: false, + storeFlag: true, + }, + 'oxigraph-worker': { + kind: 'retired', + adapter: false, + retired: true, + default: false, + wizard: false, + storeFlag: false, + }, +} as const; + +export type StoreBackend = keyof typeof STORE_BACKENDS; +export type StoreBackendPolicy = (typeof STORE_BACKENDS)[StoreBackend]; +export type StoreBackendKind = StoreBackendPolicy['kind']; +export type StoreBackendOfKind = { + [Backend in StoreBackend]: typeof STORE_BACKENDS[Backend] extends { kind: Kind } + ? Backend + : never; +}[StoreBackend]; +export type ConfigStoreBackend = { + [Backend in StoreBackend]: typeof STORE_BACKENDS[Backend] extends { retired: false } + ? Backend + : never; +}[StoreBackend]; +export type RetiredStoreBackend = { + [Backend in StoreBackend]: typeof STORE_BACKENDS[Backend] extends { retired: true } + ? Backend + : never; +}[StoreBackend]; +export type DefaultStoreBackend = { + [Backend in StoreBackend]: typeof STORE_BACKENDS[Backend] extends { default: true } + ? Backend + : never; +}[StoreBackend]; +export type WizardStoreBackend = { + [Backend in StoreBackend]: typeof STORE_BACKENDS[Backend] extends { wizard: true } + ? Backend + : never; +}[StoreBackend]; +export type StoreFlagBackend = { + [Backend in StoreBackend]: typeof STORE_BACKENDS[Backend] extends { storeFlag: true } + ? Backend + : never; +}[StoreBackend]; +export type ExternalStoreBackend = Extract, StorageAdapterBackend>; +export type LocalStoreBackend = Extract, StorageAdapterBackend>; +export type ManagedLocalStoreBackend = StoreBackendOfKind<'managed-local'>; + +export function storeBackendNames(): StoreBackend[] { + return Object.keys(STORE_BACKENDS) as StoreBackend[]; +} + +function requireSingleBackend( + backends: readonly Backend[], + description: string, +): Backend { + if (backends.length !== 1) { + throw new Error(`Expected exactly one ${description} store backend, found ${backends.length}`); + } + return backends[0]; +} + +export const DEFAULT_DAEMON_STORE_BACKEND: DefaultStoreBackend = requireSingleBackend( + storeBackendNames().filter( + (backend): backend is DefaultStoreBackend => STORE_BACKENDS[backend].default, + ), + 'default', +); + +export const MANAGED_DAEMON_STORE_BACKEND: ManagedLocalStoreBackend = requireSingleBackend( + storeBackendNames().filter( + (backend): backend is ManagedLocalStoreBackend => STORE_BACKENDS[backend].kind === 'managed-local', + ), + 'managed-local', +); + +export const DEFAULT_STORE_BACKEND = DEFAULT_DAEMON_STORE_BACKEND; +export const MANAGED_LOCAL_STORE_BACKEND = MANAGED_DAEMON_STORE_BACKEND; + +export function configBackendNames(): ConfigStoreBackend[] { + return storeBackendNames().filter( + (backend): backend is ConfigStoreBackend => !STORE_BACKENDS[backend].retired, + ); +} + +export function retiredBackendNames(): RetiredStoreBackend[] { + return storeBackendNames().filter( + (backend): backend is RetiredStoreBackend => STORE_BACKENDS[backend].retired, + ); +} + +export function configBackendList(separator = ', '): string { + return configBackendNames().join(separator); +} + +export function wizardBackendChoices(): WizardStoreBackend[] { + return storeBackendNames().filter( + (backend): backend is WizardStoreBackend => + !STORE_BACKENDS[backend].retired && STORE_BACKENDS[backend].wizard, + ); +} + +export function storeFlagBackendNames(): StoreFlagBackend[] { + return storeBackendNames().filter( + (backend): backend is StoreFlagBackend => + !STORE_BACKENDS[backend].retired && STORE_BACKENDS[backend].storeFlag, + ); +} + +export function storeFlagBackendList(separator = ', '): string { + return storeFlagBackendNames().join(separator); +} + +export function isKnownStoreBackend( + backend: string | undefined | null, +): backend is StoreBackend { + return backend != null && Object.prototype.hasOwnProperty.call(STORE_BACKENDS, backend); +} + +export function getStoreBackendPolicy( + backend: string | undefined | null, +): StoreBackendPolicy | undefined { + return isKnownStoreBackend(backend) ? STORE_BACKENDS[backend] : undefined; +} + +export function isConfigStoreBackend( + backend: string | undefined | null, +): backend is ConfigStoreBackend { + return isKnownStoreBackend(backend) && !STORE_BACKENDS[backend].retired; +} + +export function isStoreFlagBackend( + backend: string | undefined | null, +): backend is StoreFlagBackend { + return isKnownStoreBackend(backend) + && !STORE_BACKENDS[backend].retired + && STORE_BACKENDS[backend].storeFlag; +} + +export function isRetiredStoreBackend( + backend: string | undefined | null, +): backend is RetiredStoreBackend { + return isKnownStoreBackend(backend) && STORE_BACKENDS[backend].retired; +} + +export function isManagedLocalBackend( + backend: string | undefined | null, +): backend is ManagedLocalStoreBackend { + return isKnownStoreBackend(backend) && STORE_BACKENDS[backend].kind === 'managed-local'; +} + +export function isExternalStoreBackend( + backend: string | undefined | null, +): backend is ExternalStoreBackend { + return isExternalBackend(backend); +} + +/** Cross from daemon runtime config into the storage factory's adapter type. */ +export function requireStorageAdapterBackend(backend: string): StorageAdapterBackend { + if (!isStorageAdapterBackend(backend)) { + throw new Error( + `Daemon runtime store backend "${backend}" is not a constructible storage adapter`, + ); + } + return backend; +} + +export { isStorageAdapterBackend }; +export type { StorageAdapterBackend }; diff --git a/packages/cli/src/store-wizard.ts b/packages/cli/src/store-wizard.ts index 4eac50b144..768294bc71 100644 --- a/packages/cli/src/store-wizard.ts +++ b/packages/cli/src/store-wizard.ts @@ -26,6 +26,21 @@ import { type ProvisionBlazegraphDockerOptions, type ProvisionBlazegraphDockerResult, } from './daemon/blazegraph-docker.js'; +import { + DEFAULT_STORE_BACKEND, + MANAGED_LOCAL_STORE_BACKEND, + STORE_BACKENDS, + configBackendList, + isConfigStoreBackend, + isRetiredStoreBackend, + isStoreFlagBackend, + storeFlagBackendList, + wizardBackendChoices, + type ConfigStoreBackend, + type ExternalStoreBackend, + type StoreFlagBackend, + type WizardStoreBackend, +} from './store-backends.js'; export interface PromptStoreBackendOptions { /** `ask` callback that closes over a shared readline interface. */ @@ -67,9 +82,9 @@ export interface PromptStoreBackendOptions { export interface PromptStoreBackendResult { /** - * Persisted store block. `null` means "use the local default" (caller - * should omit the field or set it to undefined; saveConfig drops - * undefined keys). + * Persisted store block. `null` means "leave the store block omitted"; + * daemon boot treats an omitted store block as the managed `oxigraph-server` + * default. * * `managedByDkg: true` is set by the Docker provisioner branch only; * manual URLs always get `managedByDkg: false`. The chain-reset-wipe @@ -80,15 +95,8 @@ export interface PromptStoreBackendResult { storeBlock: ExternalStoreBlock | LocalStoreBlock | null; } -// An explicit embedded/local store block carried through verbatim on an -// Enter-through re-init. `null` still means "no block — runtime default", but -// when a node already pinned a local backend with custom `options` (e.g. the -// `options.path` the oxigraph-worker adapter reads for its persistence file), -// returning that block instead of `null` keeps re-init idempotent: cli init -// writes `store: storeBlock ?? undefined`, so a `null` would clear the block -// and relocate the store on the next boot. export type LocalStoreBlock = { - backend: 'oxigraph' | 'oxigraph-worker' | 'oxigraph-persistent'; + backend: 'oxigraph' | 'oxigraph-persistent'; options?: Record; }; @@ -110,7 +118,7 @@ export type ExternalStoreBlock = // endpoints at boot. Operator-set overrides (`port`/`location`/`cacheDir`) // that planManagedOxigraph reads at boot are carried through unchanged. | { - backend: 'oxigraph-server'; + backend: typeof MANAGED_LOCAL_STORE_BACKEND; options: Record; }; @@ -136,6 +144,39 @@ function externalStoreBlock( }; } +function isSupportedExistingBackend(backend: string | undefined): backend is ConfigStoreBackend { + return isConfigStoreBackend(backend); +} + +function retiredBackendError(source: string, backend: string): Error { + const choices = source === '--store' ? storeFlagBackendList() : configBackendList(); + return new Error( + `${source} "${backend}" is no longer supported. ` + + `Use one of: ${choices}.`, + ); +} + +function unknownBackendError(source: 'prompt' | '--store', backend: string): Error { + if (source === '--store') { + return new Error(`--store must be one of: ${storeFlagBackendList()} (got "${backend}")`); + } + return new Error(`Unknown store backend "${backend}". Expected one of: ${configBackendList()}.`); +} + +function parseBackendAnswer( + input: string, + defaultBackend: string, + choices: readonly WizardStoreBackend[], +): string { + return /^\d+$/.test(input) + ? (choices[parseInt(input, 10) - 1] ?? defaultBackend) + : input.toLowerCase(); +} + +function hasStorePath(store: PromptStoreBackendOptions['existingStore'] | undefined): boolean { + return typeof store?.options?.path === 'string' && store.options.path.trim().length > 0; +} + export async function promptStoreBackend( opts: PromptStoreBackendOptions, ): Promise { @@ -156,35 +197,24 @@ export async function promptStoreBackend( : undefined; // `oxigraph-server` (daemon-managed local RocksDB server) is the default - // local backend: it gives MVCC concurrent reads + incremental persistence, - // whereas `oxigraph` (the embedded in-process worker) rewrites the whole - // N-Quads dump on every flush. The in-process worker stays available as a - // minimal-footprint / single-reader option. NOTE: this only changes what a - // *fresh / block-less* `dkg init` writes — the runtime fallback for configs - // with no `store` block stays `oxigraph-worker`, so the existing fleet keeps - // booting unchanged on auto-update (only an explicit re-init flips a node, - // and the daemon's STORE-SWITCH guard makes that an opt-in, not silent). - // Keep ANY explicit existing backend as the default answer — including the - // embedded `oxigraph` / `oxigraph-worker` / `oxigraph-persistent` variants. - // Keeping the *exact* variant (rather than normalising the worker variants - // onto the listed `oxigraph` choice) matters for distinguishing intent: an - // Enter-through resolves the default back to that exact backend (so the - // preserve branch below keeps the block + custom options), whereas explicitly - // picking option `2` ("oxigraph") resolves to a *different* answer and is - // treated as a real switch to the plain embedded worker. Only a truly absent - // store config (fresh install / block-less node) falls through to the new - // `oxigraph-server` default. - const defaultBackend = opts.flagBackend - ?? (existingBackend === 'blazegraph' || existingBackend === 'sparql-http' || existingBackend === 'oxigraph-server' - || existingBackend === 'oxigraph' || existingBackend === 'oxigraph-worker' || existingBackend === 'oxigraph-persistent' + // local backend: it gives MVCC concurrent reads + incremental persistence. + // The old `oxigraph-worker` fallback is retired; configs that still name it + // are not preserved on Enter-through. + if (isRetiredStoreBackend(existingBackend)) { + log(` Existing store.backend "${existingBackend}" is retired; defaulting to oxigraph-server.`); + } + const flagBackend = opts.flagBackend?.trim().toLowerCase(); + if (flagBackend && !isStoreFlagBackend(flagBackend)) { + if (isRetiredStoreBackend(flagBackend)) { + throw retiredBackendError('--store', flagBackend); + } + throw unknownBackendError('--store', flagBackend); + } + const defaultBackend = flagBackend + ?? (isSupportedExistingBackend(existingBackend) ? existingBackend - : 'oxigraph-server'); - const backendChoices = ['oxigraph-server', 'oxigraph', 'blazegraph'] as const; - const backendLabels: Record = { - 'oxigraph-server': 'oxigraph-server (managed local server — recommended)', - 'oxigraph': 'oxigraph (embedded in-process worker)', - 'blazegraph': 'blazegraph (external SPARQL endpoint)', - }; + : DEFAULT_STORE_BACKEND); + const backendChoices = wizardBackendChoices(); // `sparql-http` is intentionally not listed (advanced bring-your-own-server // option) but is still accepted when typed or inherited from an existing // config / `--store` flag. Resolve the default *answer* by name for unlisted @@ -198,7 +228,7 @@ export async function promptStoreBackend( log(' Triple store backend:'); for (let i = 0; i < backendChoices.length; i++) { const choice = backendChoices[i]; - log(` ${i + 1}) ${backendLabels[choice] ?? choice}`); + log(` ${i + 1}) ${STORE_BACKENDS[choice].label}`); } // When the inherited/flagged backend isn't one of the numbered choices // (e.g. `sparql-http`), spell out that pressing Enter keeps it and that a @@ -216,49 +246,52 @@ export async function promptStoreBackend( // out-of-range number (typo like "4") falls back to `defaultBackend` — // i.e. the recommended option shown to the operator — rather than a // hard-coded `oxigraph`, so a fat-fingered digit on a fresh install no - // longer silently downgrades the node to the embedded worker. - const backendAnswer = /^\d+$/.test(backendInput) - ? (backendChoices[parseInt(backendInput, 10) - 1] ?? defaultBackend) - : backendInput.toLowerCase(); + // longer silently downgrades the node to an embedded backend. + const backendAnswer = parseBackendAnswer(backendInput, defaultBackend, backendChoices); + + if (isRetiredStoreBackend(backendAnswer)) { + throw retiredBackendError('store backend', backendAnswer); + } + if (!isSupportedExistingBackend(backendAnswer)) { + throw unknownBackendError('prompt', backendAnswer); + } + const backendPolicy = STORE_BACKENDS[backendAnswer]; // `oxigraph-server` (daemon-managed local server) is the default numbered // choice and is also accepted by name. No URL prompt or probe: the endpoint // doesn't exist until the daemon spawns it at boot. - if (backendAnswer === 'oxigraph-server') { + if (backendPolicy.kind === 'managed-local') { log(' Using a daemon-managed local Oxigraph server (started on first daemon boot).'); // Preserve existing managed-server overrides (port/location/cacheDir) on an // Enter-through: `dkg init` persists this block, so returning empty options // would silently reset a custom port/RocksDB path on the next boot — the // same hazard applyStoreFlagsToConfig guards against on the `--store` path. const prevOptions = - existingBackend === 'oxigraph-server' && opts.existingStore?.options + existingBackend === MANAGED_LOCAL_STORE_BACKEND && opts.existingStore?.options ? opts.existingStore.options : {}; - return { storeBlock: { backend: 'oxigraph-server', options: prevOptions } }; + return { storeBlock: { backend: MANAGED_LOCAL_STORE_BACKEND, options: prevOptions } }; } - if (backendAnswer !== 'blazegraph' && backendAnswer !== 'sparql-http') { - // Embedded in-process worker. Preserve an existing explicit local store - // block verbatim ONLY when the operator kept that same backend — i.e. the - // resolved answer equals the existing backend (an Enter-through, which - // resolves the default back to the exact existing variant). That keeps a - // re-init idempotent and never drops custom `options` (e.g. the worker's - // `options.path`) or relocates the store. An *explicit* switch — picking - // option `2`/"oxigraph" on a node currently using `oxigraph-worker` / - // `oxigraph-persistent`, a switch from an external/server backend, or a - // fresh / block-less init — resolves to a different answer and falls - // through to `null`, so `dkg init` clears the old block as intended. - if ( - (backendAnswer === 'oxigraph' || - backendAnswer === 'oxigraph-worker' || - backendAnswer === 'oxigraph-persistent') && - backendAnswer === existingBackend - ) { - return { storeBlock: { backend: existingBackend, options: opts.existingStore?.options } }; + if (backendPolicy.kind === 'local') { + const localBackend = backendAnswer as LocalStoreBlock['backend']; + if (STORE_BACKENDS[localBackend].requiresExistingPath && !hasStorePath(opts.existingStore)) { + throw new Error( + 'store backend "oxigraph-persistent" requires store.options.path; ' + + 'set it manually in config.json or use "oxigraph-server".', + ); } - return { storeBlock: null }; + const prevOptions = + localBackend === existingBackend && opts.existingStore?.options + ? opts.existingStore.options + : {}; + return { storeBlock: { backend: localBackend, options: prevOptions } }; + } + + if (backendPolicy.kind !== 'external') { + throw unknownBackendError('prompt', backendAnswer); } - const backend = backendAnswer as 'blazegraph' | 'sparql-http'; + const backend = backendAnswer as ExternalStoreBackend; // URL prompt loop: validate each attempt, surface the operator-facing // failure message, allow retry or abort. @@ -317,14 +350,13 @@ export async function promptStoreBackend( } const retry = (await opts.ask('Retry with a URL? (y/n)', 'y')).toLowerCase(); if (retry === 'n') { - log(' Aborting store setup; defaulting to local Oxigraph.'); + log(' Aborting store setup; using the oxigraph-server default.'); return { storeBlock: null }; } continue; } - const optionsForProbe = - backend === 'blazegraph' ? { url } : { queryEndpoint: url }; + const optionsForProbe = { [backendPolicy.queryEndpointOption]: url }; const health = await checkExternalStoreReachable({ storeConfig: { backend, options: optionsForProbe }, fetch: opts.fetch, @@ -352,7 +384,7 @@ export async function promptStoreBackend( 'y', )).toLowerCase(); if (retry === 'n') { - log(' Aborting store setup; defaulting to local Oxigraph.'); + log(' Aborting store setup; using the oxigraph-server default.'); return { storeBlock: null }; } } @@ -391,61 +423,64 @@ export async function applyStoreFlagsToConfig( opts: ApplyStoreFlagsOptions, ): Promise { const log = opts.log ?? console.log; - const backend = opts.storeFlag; + const backend = opts.storeFlag?.trim().toLowerCase(); if (!backend) return; const load = opts.loadConfig ?? loadConfig; const save = opts.saveConfig ?? saveConfig; - // Operators who pass `--store oxigraph` may be trying to FORCE local - // even though their existing config has a `store` block — honour - // that by clearing the block. - if ( - backend === 'oxigraph' || - backend === 'oxigraph-worker' || - backend === 'oxigraph-persistent' - ) { + if (isRetiredStoreBackend(backend)) { + throw retiredBackendError('--store', backend); + } + if (!isStoreFlagBackend(backend)) { + throw unknownBackendError('--store', backend); + } + const backendPolicy = STORE_BACKENDS[backend]; + + // Operators who pass `--store oxigraph` are explicitly opting into the + // embedded development store. Persist it because an omitted store block now + // means the managed `oxigraph-server` default. + if (backendPolicy.kind === 'local') { + const localBackend = backend as Extract; const existing = await load(); - if (existing.store) { - log(` Removing existing store block (--store ${backend} → local default).`); - const next = { ...existing }; - delete next.store; - await save(next); + await save({ ...existing, store: { backend: localBackend, options: {} } }); + if (localBackend === 'oxigraph') { + log(' Store configured: oxigraph (embedded development store).'); + } else { + log(` Store configured: ${localBackend}.`); } return; } // Daemon-managed local Oxigraph server: no URL to validate (the daemon // brings it up at boot). Write the block and return. - if (backend === 'oxigraph-server') { + if (backendPolicy.kind === 'managed-local') { const existing = await load(); // Preserve any existing managed-server overrides (port/location/cacheDir) // that planManagedOxigraph reads at boot — re-running setup with // `--store oxigraph-server` must not silently reset them to defaults. const prevOptions = - existing.store?.backend === 'oxigraph-server' && existing.store.options + existing.store?.backend === MANAGED_LOCAL_STORE_BACKEND && existing.store.options ? existing.store.options : {}; - await save({ ...existing, store: { backend: 'oxigraph-server', options: prevOptions } }); + await save({ ...existing, store: { backend: MANAGED_LOCAL_STORE_BACKEND, options: prevOptions } }); log(' Store configured: oxigraph-server (daemon-managed local server).'); return; } - if (backend !== 'blazegraph' && backend !== 'sparql-http') { - throw new Error( - `--store must be one of: oxigraph, blazegraph, sparql-http, oxigraph-server (got "${backend}")`, - ); + if (backendPolicy.kind !== 'external') { + throw unknownBackendError('--store', backend); } + const externalBackend = backend as ExternalStoreBackend; const url = opts.storeUrlFlag?.trim(); if (!url) { - throw new Error(`--store ${backend} requires --store-url `); + throw new Error(`--store ${externalBackend} requires --store-url `); } - const optionsForProbe = - backend === 'blazegraph' ? { url } : { queryEndpoint: url }; + const optionsForProbe = { [backendPolicy.queryEndpointOption]: url }; const health = await checkExternalStoreReachable({ - storeConfig: { backend, options: optionsForProbe }, + storeConfig: { backend: externalBackend, options: optionsForProbe }, fetch: opts.fetch, }); if (!health.ok) { @@ -455,8 +490,8 @@ export async function applyStoreFlagsToConfig( const existing = await load(); const next: DkgConfig = { ...existing, - store: externalStoreBlock(backend, url, false), + store: externalStoreBlock(externalBackend, url, false), }; await save(next); - log(` Store configured: ${backend} (${url}) — verified reachable.`); + log(` Store configured: ${externalBackend} (${url}) — verified reachable.`); } diff --git a/packages/cli/test/chain-reset-wipe.test.ts b/packages/cli/test/chain-reset-wipe.test.ts index 909ebce67b..6db6e56d4d 100644 --- a/packages/cli/test/chain-reset-wipe.test.ts +++ b/packages/cli/test/chain-reset-wipe.test.ts @@ -589,7 +589,7 @@ describe('chainResetWipe — external SPARQL wipe', () => { expect(result.failedFiles.some((f) => f.error.includes('ECONNREFUSED'))).toBe(true); }); - it('skips external wipe entirely for local backends (storeConfig present, backend oxigraph-worker)', async () => { + it('skips external wipe entirely for local backends (storeConfig present, backend oxigraph-persistent)', async () => { writeFileSync(join(dataDir, 'store.nq'), '

.'); const { fn, calls } = mockFetch(() => new Response(null, { status: 200 })); @@ -597,7 +597,7 @@ describe('chainResetWipe — external SPARQL wipe', () => { dataDir, currentMarker: NEW_MARKER, storeConfig: { - backend: 'oxigraph-worker', + backend: 'oxigraph-persistent', // Options that LOOK like an external URL: these must be ignored // because the backend itself is local. Otherwise an operator who // hand-tuned options would see surprise SPARQL requests. @@ -652,7 +652,7 @@ describe('detectBackendSwitch', () => { const logs: string[] = []; const result = detectBackendSwitch({ dataDir, - currentBackend: 'oxigraph-worker', + currentBackend: 'oxigraph-persistent', acceptStoreReset: false, log: (m) => logs.push(m), }); @@ -662,7 +662,7 @@ describe('detectBackendSwitch', () => { expect(result.aborted).toBe(false); const persisted = JSON.parse(readFileSync(join(dataDir, STATE_FILE), 'utf8')); - expect(persisted.lastBackend).toBe('oxigraph-worker'); + expect(persisted.lastBackend).toBe('oxigraph-persistent'); // First-boot path is silent — no STORE-SWITCH warning header. expect(logs.find((l) => l.includes('STORE-SWITCH'))).toBeUndefined(); }); @@ -696,19 +696,19 @@ describe('detectBackendSwitch', () => { it('is a no-op when backend matches previous boot', () => { writeFileSync( join(dataDir, STATE_FILE), - JSON.stringify({ chainResetMarker: null, lastBackend: 'oxigraph-worker', savedAt: Date.now() }), + JSON.stringify({ chainResetMarker: null, lastBackend: 'oxigraph-persistent', savedAt: Date.now() }), ); const logs: string[] = []; const result = detectBackendSwitch({ dataDir, - currentBackend: 'oxigraph-worker', + currentBackend: 'oxigraph-persistent', acceptStoreReset: false, log: (m) => logs.push(m), }); expect(result.changed).toBe(false); - expect(result.previous).toBe('oxigraph-worker'); + expect(result.previous).toBe('oxigraph-persistent'); expect(result.aborted).toBe(false); expect(logs).toEqual([]); }); @@ -716,7 +716,7 @@ describe('detectBackendSwitch', () => { it('aborts boot on mismatch without acceptStoreReset, surfaces multi-line warning', () => { writeFileSync( join(dataDir, STATE_FILE), - JSON.stringify({ chainResetMarker: null, lastBackend: 'oxigraph-worker', savedAt: Date.now() }), + JSON.stringify({ chainResetMarker: null, lastBackend: 'oxigraph-persistent', savedAt: Date.now() }), ); const logs: string[] = []; @@ -728,25 +728,25 @@ describe('detectBackendSwitch', () => { }); expect(result.changed).toBe(true); - expect(result.previous).toBe('oxigraph-worker'); + expect(result.previous).toBe('oxigraph-persistent'); expect(result.aborted).toBe(true); const joined = logs.join('\n'); expect(joined).toMatch(/STORE-SWITCH/); - expect(joined).toMatch(/previous: oxigraph-worker/); + expect(joined).toMatch(/previous: oxigraph-persistent/); expect(joined).toMatch(/current:\s+blazegraph/); expect(joined).toMatch(/DKG_ACCEPT_STORE_RESET=1/); // State file MUST still record the old backend so a corrected // config (operator reverts the edit) sees a match on next boot. const persisted = JSON.parse(readFileSync(join(dataDir, STATE_FILE), 'utf8')); - expect(persisted.lastBackend).toBe('oxigraph-worker'); + expect(persisted.lastBackend).toBe('oxigraph-persistent'); }); it('proceeds and updates state when acceptStoreReset=true', () => { writeFileSync( join(dataDir, STATE_FILE), - JSON.stringify({ chainResetMarker: null, lastBackend: 'oxigraph-worker', savedAt: Date.now() }), + JSON.stringify({ chainResetMarker: null, lastBackend: 'oxigraph-persistent', savedAt: Date.now() }), ); const logs: string[] = []; @@ -775,12 +775,12 @@ describe('detectBackendSwitch', () => { // Establish a baseline by running detectBackendSwitch first. detectBackendSwitch({ dataDir, - currentBackend: 'oxigraph-worker', + currentBackend: 'oxigraph-persistent', acceptStoreReset: false, }); expect( JSON.parse(readFileSync(join(dataDir, STATE_FILE), 'utf8')).lastBackend, - ).toBe('oxigraph-worker'); + ).toBe('oxigraph-persistent'); // Now run a marker change — the wipe path persists chainResetMarker // and MUST preserve lastBackend. @@ -792,7 +792,7 @@ describe('detectBackendSwitch', () => { const persisted = JSON.parse(readFileSync(join(dataDir, STATE_FILE), 'utf8')); expect(persisted.chainResetMarker).toBe(NEW_MARKER); - expect(persisted.lastBackend).toBe('oxigraph-worker'); + expect(persisted.lastBackend).toBe('oxigraph-persistent'); }); }); diff --git a/packages/cli/test/daemon-http-behavior-extra.test.ts b/packages/cli/test/daemon-http-behavior-extra.test.ts index c4f6a30152..3f2e3a10b4 100644 --- a/packages/cli/test/daemon-http-behavior-extra.test.ts +++ b/packages/cli/test/daemon-http-behavior-extra.test.ts @@ -102,7 +102,7 @@ async function writeDaemonConfig( relay: 'none', auth: { enabled: authEnabled }, store: { - backend: 'oxigraph-worker', + backend: 'oxigraph-persistent', options: { path: join(home, 'store.nq') }, }, // Real EVM adapter against the shared Hardhat node (port 9548 per diff --git a/packages/cli/test/daemon-http-inflight-cap.test.ts b/packages/cli/test/daemon-http-inflight-cap.test.ts index e38b58c48f..c36f9b0749 100644 --- a/packages/cli/test/daemon-http-inflight-cap.test.ts +++ b/packages/cli/test/daemon-http-inflight-cap.test.ts @@ -1,4 +1,5 @@ import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { request as httpRequest } from 'node:http'; import { startLiveDaemon, stopLiveDaemon, authHeaders, type LiveDaemon } from './helpers/live-daemon.js'; /** @@ -11,13 +12,11 @@ import { startLiveDaemon, stopLiveDaemon, authHeaders, type LiveDaemon } from '. * http-admission-control.test.ts: it would fail if the limiter were never wired * into createServer, wired after an early return, or never released. * - * Saturation is created with a 50-request burst against cap=1 rather than a - * single held-open request. That is statistically deterministic — 50 concurrent - * requests cannot all serialize through one slot without overlap — and avoids a - * brittle blocking fixture (the daemon admits before the route reads the body, - * so an unfinished-body "hold" does not reliably pin the slot). The precise - * one-in/one-shed/release semantics are covered deterministically by the unit - * tests; here we prove the wiring end-to-end. + * Saturation is created with a real HTTP request whose JSON body is deliberately + * left unfinished. The test then polls the admission-exempt status route until + * it observes the occupied slot before sending competing requests. This proves + * the production wiring without relying on a fast request burst happening to + * overlap on a particular runner. */ describe('daemon admission control (real node, maxInFlightRequests=1)', () => { let daemon: LiveDaemon | undefined; @@ -46,24 +45,76 @@ describe('daemon admission control (real node, maxInFlightRequests=1)', () => { }); } + async function readAdmission( + d: LiveDaemon, + ): Promise<{ inFlight: number; max: number; rejectedTotal: number }> { + const res = await fetch(`${d.base}/api/status`, { headers: authHeaders(d) }); + expect(res.status).toBe(200); + const body = (await res.json()) as { + admission?: { inFlight: number; max: number; rejectedTotal: number }; + }; + expect(body.admission).toBeDefined(); + return body.admission!; + } + + async function holdQuerySlot(d: LiveDaemon): Promise<{ + release: () => void; + responseStatus: Promise; + }> { + let released = false; + let req: ReturnType; + const responseStatus = new Promise((resolve, reject) => { + req = httpRequest( + `${d.base}/api/query`, + { + method: 'POST', + headers: authHeaders(d), + }, + (res) => { + res.resume(); + res.once('end', () => resolve(res.statusCode ?? 0)); + }, + ); + req.once('error', reject); + req.write('{"sparql":"SELECT * WHERE { ?s ?p ?o } LIMIT 1","hold":"'); + }); + const release = () => { + if (released) return; + released = true; + req.end('released"}'); + }; + + const deadline = Date.now() + 5_000; + while (Date.now() < deadline) { + if ((await readAdmission(d)).inFlight === 1) { + return { release, responseStatus }; + } + await new Promise((resolve) => setTimeout(resolve, 20)); + } + release(); + throw new Error('Timed out waiting for the held query to occupy the admission slot'); + } + it('sheds concurrent over-capacity requests with 503 + Retry-After, then recovers', async () => { const d = daemon!; - const results = await Promise.all( - Array.from({ length: 50 }, () => - selectQuery(d) - .then((r) => ({ status: r.status, retryAfter: r.headers.get('retry-after') })) - .catch(() => ({ status: 0, retryAfter: null as string | null })), - ), - ); - const shed = results.filter((r) => r.status === 503); - const ok = results.filter((r) => r.status === 200); - - // Every result must be an EXPECTED status — never a network error (0) or an - // unexpected 4xx/5xx that would otherwise hide behind the >=1/>=1 counts. - expect(results.every((r) => r.status === 200 || r.status === 503)).toBe(true); - expect(ok.length).toBeGreaterThan(0); // at least one admitted - expect(shed.length).toBeGreaterThan(0); // cap enforced under concurrent load - expect(shed.every((r) => r.retryAfter === '1')).toBe(true); // Retry-After present on every 503 + const held = await holdQuerySlot(d); + try { + const results = await Promise.all( + Array.from({ length: 10 }, () => + selectQuery(d) + .then((r) => ({ status: r.status, retryAfter: r.headers.get('retry-after') })) + .catch(() => ({ status: 0, retryAfter: null as string | null })), + ), + ); + + expect(results.every((r) => r.status === 503)).toBe(true); + expect(results.every((r) => r.retryAfter === '1')).toBe(true); + + held.release(); + expect(await held.responseStatus).toBe(200); + } finally { + held.release(); + } // Slots are released after each handler completes → a fresh request succeeds. const recovered = await selectQuery(d); @@ -72,58 +123,54 @@ describe('daemon admission control (real node, maxInFlightRequests=1)', () => { it('keeps the exempt liveness path (/api/status) answerable even while saturated', async () => { const d = daemon!; - // Saturate with non-exempt query work; capture the burst results so we can - // PROVE the daemon was actually over capacity (>=1 shed) while the status - // probes ran — otherwise "status stayed 200" would be vacuous. - const burst = Promise.all( - Array.from({ length: 40 }, () => - selectQuery(d).then((r) => r.status).catch(() => 0), - ), - ); - // ...while hammering the exempt status endpoint, which must always answer 200. - const statuses = await Promise.all( - Array.from({ length: 12 }, () => - fetch(`${d.base}/api/status`, { headers: authHeaders(d) }) - .then((r) => r.status) - .catch(() => 0), - ), - ); - const burstStatuses = await burst; - - expect(statuses.every((s) => s === 200)).toBe(true); // exempt path never shed - expect(burstStatuses.filter((s) => s === 503).length).toBeGreaterThan(0); // saturation really happened - expect(burstStatuses.every((s) => s === 200 || s === 503)).toBe(true); // no unexpected failures + const held = await holdQuerySlot(d); + try { + const [statuses, burstStatuses] = await Promise.all([ + Promise.all( + Array.from({ length: 12 }, () => + fetch(`${d.base}/api/status`, { headers: authHeaders(d) }) + .then((r) => r.status) + .catch(() => 0), + ), + ), + Promise.all( + Array.from({ length: 10 }, () => + selectQuery(d).then((r) => r.status).catch(() => 0), + ), + ), + ]); + + expect(statuses.every((s) => s === 200)).toBe(true); + expect(burstStatuses.every((s) => s === 503)).toBe(true); + } finally { + held.release(); + } + expect(await held.responseStatus).toBe(200); }, 60_000); it('surfaces admission stats on /api/status (effective cap + per-burst shed delta)', async () => { const d = daemon!; - // Read the surfaced admission block off the exempt status endpoint. - const readAdmission = async (): Promise<{ inFlight: number; max: number; rejectedTotal: number }> => { - const res = await fetch(`${d.base}/api/status`, { headers: authHeaders(d) }); - expect(res.status).toBe(200); - const body = (await res.json()) as { - admission?: { inFlight: number; max: number; rejectedTotal: number }; - }; - expect(body.admission).toBeDefined(); - return body.admission!; - }; - // Snapshot BEFORE this burst — earlier tests in this file already shed, so a // bare `rejectedTotal > 0` would pass without proving THIS burst moved the // counter (i.e. that the surfaced value still tracks live shedding). - const before = await readAdmission(); + const before = await readAdmission(d); expect(before.max).toBe(1); // the pinned effective cap is surfaced expect(typeof before.inFlight).toBe('number'); - // Saturate the non-exempt path so this burst provably sheds. - const burst = await Promise.all( - Array.from({ length: 50 }, () => selectQuery(d).then((r) => r.status).catch(() => 0)), - ); - expect(burst.filter((s) => s === 503).length).toBeGreaterThan(0); // this burst really shed + const held = await holdQuerySlot(d); + try { + const burst = await Promise.all( + Array.from({ length: 10 }, () => selectQuery(d).then((r) => r.status).catch(() => 0)), + ); + expect(burst.every((s) => s === 503)).toBe(true); + } finally { + held.release(); + } + expect(await held.responseStatus).toBe(200); // /api/status is admission-exempt, so reading it doesn't perturb the counter: // `after` MUST exceed `before` by the sheds we just caused. - const after = await readAdmission(); + const after = await readAdmission(d); expect(after.rejectedTotal).toBeGreaterThan(before.rejectedTotal); }, 60_000); }); diff --git a/packages/cli/test/daemon-startup-validation.test.ts b/packages/cli/test/daemon-startup-validation.test.ts index cef225b46b..cee7688fb6 100644 --- a/packages/cli/test/daemon-startup-validation.test.ts +++ b/packages/cli/test/daemon-startup-validation.test.ts @@ -1,5 +1,5 @@ -import { afterEach, describe, expect, it, vi } from 'vitest'; -import { mkdtemp, rm } from 'node:fs/promises'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { computeNetworkId } from '../../core/src/genesis.js'; @@ -9,6 +9,9 @@ const mocks = vi.hoisted(() => ({ agentCreate: vi.fn(), loadOpWallets: vi.fn(), loadNetworkConfig: vi.fn(), + checkExternalStoreReachable: vi.fn(), + checkOrSetStoreIdentity: vi.fn(), + startManagedOxigraph: vi.fn(), })); vi.mock('@origintrail-official/dkg-agent', async importOriginal => { @@ -29,6 +32,23 @@ vi.mock('../src/config.js', async importOriginal => { }; }); +vi.mock('../src/daemon/oxigraph-managed.js', async importOriginal => { + const actual = await importOriginal(); + return { + ...actual, + startManagedOxigraph: mocks.startManagedOxigraph, + }; +}); + +vi.mock('../src/daemon/store-health-check.js', async importOriginal => { + const actual = await importOriginal(); + return { + ...actual, + checkExternalStoreReachable: mocks.checkExternalStoreReachable, + checkOrSetStoreIdentity: mocks.checkOrSetStoreIdentity, + }; +}); + const { runDaemonInner } = await import('../src/daemon/lifecycle.js'); function closeDashboardDbFromAgentCreateArg(createArg: any): void { @@ -38,6 +58,29 @@ function closeDashboardDbFromAgentCreateArg(createArg: any): void { db?.close?.(); } +function managedOxigraphResult(dataDir: string) { + return { + handle: { + queryEndpoint: 'http://127.0.0.1:12001/query', + updateEndpoint: 'http://127.0.0.1:12001/update', + killSync: vi.fn(), + }, + storeConfig: { + backend: 'sparql-http', + options: { + queryEndpoint: 'http://127.0.0.1:12001/query', + updateEndpoint: 'http://127.0.0.1:12001/update', + managedByDkg: true, + }, + }, + largeLiteralStorage: { enabled: true, directory: join(dataDir, 'literal-blobs') }, + sharedMemoryPublicSnapshotStorage: { + enabled: true, + directory: join(dataDir, 'swm-public-snapshots'), + }, + }; +} + describe('daemon startup network validation', () => { let tempHome: string | undefined; let originalDkgHome: string | undefined; @@ -45,6 +88,15 @@ describe('daemon startup network validation', () => { let stderrWrite: typeof process.stderr.write = process.stderr.write; let uncaughtExceptionListeners: NodeJS.UncaughtExceptionListener[] = []; let unhandledRejectionListeners: NodeJS.UnhandledRejectionListener[] = []; + const originalAcceptStoreReset = process.env.DKG_ACCEPT_STORE_RESET; + + beforeEach(() => { + mocks.loadNetworkConfig.mockResolvedValue(undefined); + mocks.loadOpWallets.mockResolvedValue({ adminWallet: undefined, wallets: [] }); + mocks.startManagedOxigraph.mockResolvedValue(null); + mocks.checkExternalStoreReachable.mockResolvedValue({ ok: true, backend: 'sparql-http', endpoint: 'http://127.0.0.1:12001/query' }); + mocks.checkOrSetStoreIdentity.mockResolvedValue({ ok: true, action: 'matched', nodeName: 'test-node' }); + }); afterEach(async () => { vi.restoreAllMocks(); @@ -64,10 +116,142 @@ describe('daemon startup network validation', () => { } else { process.env.DKG_HOME = originalDkgHome; } + if (originalAcceptStoreReset === undefined) { + delete process.env.DKG_ACCEPT_STORE_RESET; + } else { + process.env.DKG_ACCEPT_STORE_RESET = originalAcceptStoreReset; + } if (tempHome) await rm(tempHome, { recursive: true, force: true }); tempHome = undefined; }); + async function useTempHome(prefix: string) { + tempHome = await mkdtemp(join(tmpdir(), prefix)); + originalDkgHome = process.env.DKG_HOME; + process.env.DKG_HOME = tempHome; + stdoutWrite = process.stdout.write; + stderrWrite = process.stderr.write; + uncaughtExceptionListeners = process.listeners('uncaughtException') as NodeJS.UncaughtExceptionListener[]; + unhandledRejectionListeners = process.listeners('unhandledRejection') as NodeJS.UnhandledRejectionListener[]; + return vi.spyOn(process.stdout, 'write').mockImplementation(() => true); + } + + it('exits before managed store startup when a blockless config has legacy store.nq and no reset acknowledgement', async () => { + const stdoutSpy = await useTempHome('dkg-legacy-store-gate-'); + await writeFile(join(tempHome!, 'store.nq'), '

.'); + vi + .spyOn(process, 'exit') + .mockImplementation(((code?: string | number | null) => { + throw new Error(`process.exit:${code}`); + }) as never); + + await expect(runDaemonInner(true, { + name: 'legacy-store-gate-test', + listenPort: 0, + nodeRole: 'edge', + } as any, Date.now())).rejects.toThrow('process.exit:1'); + + const output = stdoutSpy.mock.calls.map(call => String(call[0])).join(''); + expect(output).toContain('legacy store.nq from the old implicit worker default'); + expect(output).toContain('DKG_ACCEPT_STORE_RESET=1'); + expect(mocks.startManagedOxigraph).not.toHaveBeenCalled(); + expect(mocks.agentCreate).not.toHaveBeenCalled(); + }); + + it('continues with the effective oxigraph-server store after legacy store.nq is acknowledged', async () => { + const stdoutSpy = await useTempHome('dkg-legacy-store-ack-'); + process.env.DKG_ACCEPT_STORE_RESET = '1'; + await writeFile(join(tempHome!, 'store.nq'), '

.'); + mocks.startManagedOxigraph.mockResolvedValue(managedOxigraphResult(tempHome!)); + mocks.agentCreate.mockRejectedValue(new Error('after-agent-create')); + + await expect(runDaemonInner(true, { + name: 'legacy-store-ack-test', + listenPort: 0, + nodeRole: 'edge', + } as any, Date.now())).rejects.toThrow('after-agent-create'); + + const output = stdoutSpy.mock.calls.map(call => String(call[0])).join(''); + expect(output).toContain('using oxigraph-server'); + expect(mocks.startManagedOxigraph).toHaveBeenCalledTimes(1); + expect(mocks.startManagedOxigraph.mock.calls[0]?.[0]).toMatchObject({ + dataDir: tempHome, + config: { + store: { backend: 'oxigraph-server', options: {} }, + }, + }); + expect(mocks.agentCreate).toHaveBeenCalledTimes(1); + expect(mocks.agentCreate.mock.calls[0]?.[0]).toMatchObject({ + storeConfig: { + backend: 'sparql-http', + options: { + queryEndpoint: 'http://127.0.0.1:12001/query', + updateEndpoint: 'http://127.0.0.1:12001/update', + managedByDkg: true, + }, + }, + largeLiteralStorage: { enabled: true, directory: join(tempHome!, 'literal-blobs') }, + sharedMemoryPublicSnapshotStorage: { enabled: true, directory: join(tempHome!, 'swm-public-snapshots') }, + }); + }); + + it('blocks a wizard-rewritten oxigraph-server config with legacy store.nq and no backend marker', async () => { + const stdoutSpy = await useTempHome('dkg-rewritten-legacy-store-gate-'); + await writeFile(join(tempHome!, 'store.nq'), '

.'); + vi + .spyOn(process, 'exit') + .mockImplementation(((code?: string | number | null) => { + throw new Error(`process.exit:${code}`); + }) as never); + + await expect(runDaemonInner(true, { + name: 'rewritten-legacy-store-gate-test', + listenPort: 0, + nodeRole: 'edge', + store: { backend: 'oxigraph-server', options: {} }, + } as any, Date.now())).rejects.toThrow('process.exit:1'); + + const output = stdoutSpy.mock.calls.map(call => String(call[0])).join(''); + expect(output).toContain('legacy store.nq from the old worker-backed store'); + expect(output).toContain('DKG_ACCEPT_STORE_RESET=1'); + expect(mocks.startManagedOxigraph).not.toHaveBeenCalled(); + expect(mocks.agentCreate).not.toHaveBeenCalled(); + }); + + it('migrates an explicit legacy worker config after reset acknowledgement', async () => { + await useTempHome('dkg-explicit-legacy-store-ack-'); + process.env.DKG_ACCEPT_STORE_RESET = '1'; + await writeFile(join(tempHome!, 'store.nq'), '

.'); + mocks.startManagedOxigraph.mockResolvedValue({ + handle: { queryEndpoint: 'http://127.0.0.1:12001/query', updateEndpoint: 'http://127.0.0.1:12001/update', killSync: vi.fn() }, + storeConfig: { + backend: 'sparql-http', + options: { + queryEndpoint: 'http://127.0.0.1:12001/query', + updateEndpoint: 'http://127.0.0.1:12001/update', + managedByDkg: true, + }, + }, + largeLiteralStorage: { enabled: true, directory: join(tempHome!, 'literal-blobs') }, + sharedMemoryPublicSnapshotStorage: { enabled: true, directory: join(tempHome!, 'swm-public-snapshots') }, + }); + mocks.agentCreate.mockRejectedValue(new Error('after-agent-create')); + + await expect(runDaemonInner(true, { + name: 'explicit-legacy-store-ack-test', + listenPort: 0, + nodeRole: 'edge', + store: { backend: 'oxigraph-worker' }, + } as any, Date.now())).rejects.toThrow('after-agent-create'); + + expect(mocks.startManagedOxigraph.mock.calls[0]?.[0]).toMatchObject({ + config: { store: { backend: 'oxigraph-server', options: {} } }, + }); + expect(mocks.agentCreate.mock.calls[0]?.[0]).toMatchObject({ + storeConfig: { backend: 'sparql-http' }, + }); + }); + it('exits before agent creation when the selected network is pre-deployment', async () => { tempHome = await mkdtemp(join(tmpdir(), 'dkg-predeployment-startup-')); originalDkgHome = process.env.DKG_HOME; @@ -160,6 +344,7 @@ describe('daemon startup network validation', () => { defaultNodeRole: 'edge', }); mocks.loadOpWallets.mockResolvedValue({ adminWallet: undefined, wallets: [] }); + mocks.startManagedOxigraph.mockResolvedValue(managedOxigraphResult(tempHome)); mocks.agentCreate.mockRejectedValue(new Error('after-agent-create')); vi.spyOn(process.stdout, 'write').mockImplementation(() => true); @@ -214,6 +399,7 @@ describe('daemon startup network validation', () => { defaultNodeRole: 'edge', }); mocks.loadOpWallets.mockResolvedValue({ adminWallet: undefined, wallets: [] }); + mocks.startManagedOxigraph.mockResolvedValue(managedOxigraphResult(tempHome)); mocks.agentCreate.mockRejectedValue(new Error('after-agent-create')); vi.spyOn(process.stdout, 'write').mockImplementation(() => true); diff --git a/packages/cli/test/daemon-state.test.ts b/packages/cli/test/daemon-state.test.ts new file mode 100644 index 0000000000..0c158fc8eb --- /dev/null +++ b/packages/cli/test/daemon-state.test.ts @@ -0,0 +1,53 @@ +import { describe, expect, it } from 'vitest'; +import { mkdtemp, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + DAEMON_STATE_FILE, + readPersistedDaemonState, + readPersistedNetworkConfig, + readPersistedStoreBackend, + writePersistedChainResetMarker, + writePersistedNetworkConfig, + writePersistedStoreBackend, +} from '../src/daemon/daemon-state.js'; + +describe('persisted daemon state', () => { + it('preserves sibling fields when marker, backend, and network writers update independently', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-daemon-state-')); + + writePersistedStoreBackend(dataDir, 'oxigraph-server'); + writePersistedNetworkConfig(dataDir, 'mainnet-gnosis'); + writePersistedChainResetMarker(dataDir, 'reset-42'); + + expect(readPersistedDaemonState(dataDir)).toMatchObject({ + chainResetMarker: 'reset-42', + lastBackend: 'oxigraph-server', + lastNetworkConfig: 'mainnet-gnosis', + }); + expect(readPersistedStoreBackend(dataDir)).toBe('oxigraph-server'); + expect(readPersistedNetworkConfig(dataDir)).toBe('mainnet-gnosis'); + + writePersistedStoreBackend(dataDir, 'blazegraph'); + expect(readPersistedDaemonState(dataDir)).toMatchObject({ + chainResetMarker: 'reset-42', + lastBackend: 'blazegraph', + lastNetworkConfig: 'mainnet-gnosis', + }); + }); + + it('treats malformed state as absent and repairs it on the next write', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-daemon-state-invalid-')); + await writeFile( + join(dataDir, DAEMON_STATE_FILE), + JSON.stringify({ chainResetMarker: 42, lastBackend: 'oxigraph' }), + ); + + expect(readPersistedDaemonState(dataDir)).toBeNull(); + writePersistedNetworkConfig(dataDir, 'testnet'); + expect(readPersistedDaemonState(dataDir)).toMatchObject({ + chainResetMarker: null, + lastNetworkConfig: 'testnet', + }); + }); +}); diff --git a/packages/cli/test/daemon-storage-ack-timing-wiring.test.ts b/packages/cli/test/daemon-storage-ack-timing-wiring.test.ts index 98840f7494..5934ab3650 100644 --- a/packages/cli/test/daemon-storage-ack-timing-wiring.test.ts +++ b/packages/cli/test/daemon-storage-ack-timing-wiring.test.ts @@ -148,6 +148,7 @@ describe('runDaemonInner StorageACK timing wiring', () => { networkConfig: 'mainnet-gnosis', listenPort: 0, nodeRole: 'core', + store: { backend: 'oxigraph' }, chain: { type: 'evm', rpcUrl: 'https://private-rpc.example', @@ -285,6 +286,7 @@ describe('runDaemonInner StorageACK timing wiring', () => { listenPort: 0, nodeRole: 'edge', apiPort: 0, + store: { backend: 'oxigraph' }, auth: { enabled: false }, promoteQueue: { enabled: false }, publisher: { enabled: true }, diff --git a/packages/cli/test/daemon-sync-agents-meta-wiring.test.ts b/packages/cli/test/daemon-sync-agents-meta-wiring.test.ts index 07eb5d57b7..3f4eaa1636 100644 --- a/packages/cli/test/daemon-sync-agents-meta-wiring.test.ts +++ b/packages/cli/test/daemon-sync-agents-meta-wiring.test.ts @@ -114,6 +114,7 @@ describe('runDaemonInner wires sync options into DKGAgent.create', () => { networkConfig: 'mainnet-gnosis', listenPort: 0, nodeRole: 'core', + store: { backend: 'oxigraph' }, chain: { type: 'evm', rpcUrl: 'https://private-rpc.example', diff --git a/packages/cli/test/daemon/plugin-routes-api.e2e.test.ts b/packages/cli/test/daemon/plugin-routes-api.e2e.test.ts index c1617aa888..3204585822 100644 --- a/packages/cli/test/daemon/plugin-routes-api.e2e.test.ts +++ b/packages/cli/test/daemon/plugin-routes-api.e2e.test.ts @@ -62,7 +62,7 @@ async function writeDaemonConfig( relay: 'none', auth: { enabled: true }, store: { - backend: 'oxigraph-worker', + backend: 'oxigraph-persistent', options: { path: join(home, 'store.nq') }, }, chain: { diff --git a/packages/cli/test/handle-request-store-persistence.test.ts b/packages/cli/test/handle-request-store-persistence.test.ts new file mode 100644 index 0000000000..0afd3d0305 --- /dev/null +++ b/packages/cli/test/handle-request-store-persistence.test.ts @@ -0,0 +1,103 @@ +import { createServer } from 'node:http'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import type { AddressInfo } from 'node:net'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { DkgConfig } from '../src/config.js'; +import { handleRequest } from '../src/daemon/handle-request.js'; +import { createRequestStoreContext } from '../src/daemon/routes/context.js'; +import type { StoreRuntimeContext } from '../src/daemon/store-runtime.js'; + +describe('handleRequest store config boundary', () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('persists the operator config rather than the materialized managed-store runtime', async () => { + const home = await mkdtemp(join(tmpdir(), 'dkg-handle-request-store-')); + vi.stubEnv('DKG_HOME', home); + + const operatorConfig: DkgConfig = { + name: 'operator-config-route-test', + nodeRole: 'edge', + chain: { type: 'mock' }, + }; + const storeRuntime: StoreRuntimeContext = { + operatorConfig, + effectiveStore: { backend: 'oxigraph-server', options: {} }, + runtimeStore: { + backend: 'sparql-http', + options: { + queryEndpoint: 'http://127.0.0.1:7878/query', + updateEndpoint: 'http://127.0.0.1:7878/update', + managedByDkg: true, + }, + }, + }; + const requestStoreContext = createRequestStoreContext(storeRuntime); + expect(requestStoreContext).toEqual({ + config: operatorConfig, + effectiveStore: storeRuntime.effectiveStore, + runtimeStore: storeRuntime.runtimeStore, + }); + expect(requestStoreContext).not.toHaveProperty('operatorConfig'); + expect(requestStoreContext).not.toHaveProperty('storeRuntime'); + + const server = createServer((req, res) => { + const args: Parameters = [ + req, + res, + { resolveAgentAddress: () => 'operator-config-route-test' } as any, + {} as any, + null, + storeRuntime, + Date.now(), + {} as any, + { wallets: [] }, + null, + {} as any, + {} as any, + undefined, + '0.0.0-test', + '', + {} as any, + {} as any, + new Map(), + new Map(), + {} as any, + null, + new Set(), + '127.0.0.1', + { value: 0 }, + [], + { inFlight: 0, max: 0, rejectedTotal: 0 }, + ]; + void handleRequest(...args).catch(() => { + res.statusCode = 500; + res.end('route failed'); + }); + }); + + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + try { + const address = server.address() as AddressInfo; + const response = await fetch(`http://127.0.0.1:${address.port}/api/register-adapter`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ id: 'openclaw' }), + }); + expect(response.status).toBe(200); + + const persisted = JSON.parse(await readFile(join(home, 'config.json'), 'utf8')) as DkgConfig; + expect(persisted.store).toBeUndefined(); + expect(persisted.localAgentIntegrations?.openclaw?.enabled).toBe(true); + expect(JSON.stringify(persisted)).not.toContain('127.0.0.1:7878'); + } finally { + await new Promise((resolve, reject) => { + server.close((error) => error ? reject(error) : resolve()); + }); + await rm(home, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/cli/test/helpers/live-daemon.ts b/packages/cli/test/helpers/live-daemon.ts index 683b1ba5f9..217188bd0a 100644 --- a/packages/cli/test/helpers/live-daemon.ts +++ b/packages/cli/test/helpers/live-daemon.ts @@ -89,7 +89,7 @@ export async function startLiveDaemon(opts: StartDaemonOpts = {}): Promise ({})) as { code?: string; reason?: string }; - if (body.code !== 'async_publisher_unavailable') break; + const body = await res.json().catch(() => ({})) as { error?: string; reason?: string }; + if (body.error !== 'PublisherUnavailable' && body.error !== 'PublisherDisabled') break; if (body.reason !== 'publisher_starting') { throw new Error(`Async publisher failed readiness: ${body.reason ?? res.status}`); } diff --git a/packages/cli/test/oxigraph-managed.test.ts b/packages/cli/test/oxigraph-managed.test.ts index f069eb6676..cb89c9592f 100644 --- a/packages/cli/test/oxigraph-managed.test.ts +++ b/packages/cli/test/oxigraph-managed.test.ts @@ -86,7 +86,7 @@ afterAll(async () => { describe('planManagedOxigraph', () => { it('returns null for non-oxigraph-server backends', () => { - expect(planManagedOxigraph({ store: { backend: 'oxigraph-worker' } }, '/data')).toBeNull(); + expect(planManagedOxigraph({ store: { backend: 'oxigraph' } }, '/data')).toBeNull(); expect(planManagedOxigraph({ store: { backend: 'sparql-http' } }, '/data')).toBeNull(); expect(planManagedOxigraph({}, '/data')).toBeNull(); }); @@ -327,7 +327,7 @@ describe('startManagedOxigraph (real download + real server)', () => { // Contract: a non-managed backend is a no-op — null result, and nothing // observable happens (no cache dir is created, nothing binds a port). const result = await startManagedOxigraph({ - config: { store: { backend: 'oxigraph-worker' } }, + config: { store: { backend: 'oxigraph' } }, dataDir: '/data', }); expect(result).toBeNull(); diff --git a/packages/cli/test/publisher-managed-store.test.ts b/packages/cli/test/publisher-managed-store.test.ts new file mode 100644 index 0000000000..5e25109379 --- /dev/null +++ b/packages/cli/test/publisher-managed-store.test.ts @@ -0,0 +1,22 @@ +import { describe, expect, it } from 'vitest'; +import { mkdtemp } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { createPublisherInspector } from '../src/publisher-runner.js'; + +describe('daemon-down publisher inspection', () => { + it('requires the daemon for the managed oxigraph-server backend', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-publisher-managed-store-')); + + await expect(createPublisherInspector({ + dataDir, + config: { + name: 'managed-publisher-test', + nodeRole: 'edge', + store: { backend: 'oxigraph-server', options: {} }, + }, + })).rejects.toThrow( + /daemon-managed store "oxigraph-server" require a running DKG daemon.*Start the daemon and retry/, + ); + }); +}); diff --git a/packages/cli/test/publisher-wallets.test.ts b/packages/cli/test/publisher-wallets.test.ts index 10e2085a9b..3b73d221b3 100644 --- a/packages/cli/test/publisher-wallets.test.ts +++ b/packages/cli/test/publisher-wallets.test.ts @@ -144,6 +144,48 @@ describe('publisher wallets', () => { ).rejects.toThrow('dkg publisher wallet add '); }); + it('boots and closes the standalone publisher runtime with the persistent fallback when config has no store block', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-publisher-runtime-')); + const wallet = ethers.Wallet.createRandom(); + await addPublisherWallet(dataDir, wallet.privateKey); + + const runtime = await createPublisherRuntime({ + dataDir, + config: { + name: 'test-node', + apiPort: 9200, + listenPort: 0, + nodeRole: 'edge', + contextGraphs: [], + chain: { type: 'mock' }, + }, + pollIntervalMs: 10, + errorBackoffMs: 10, + }); + + await runtime.publisher.lift({ + swmId: 'swm-main', + shareOperationId: 'share-no-store-fallback', + roots: ['urn:local:/fallback'], + contextGraphId: 'music-social', + namespace: 'aloha', + scope: 'person-profile', + transitionType: 'CREATE', + authority: { type: 'owner', proofRef: 'proof:owner:fallback' }, + }); + + await runtime.stop(); + const persistentStore = await createTripleStore({ + backend: 'oxigraph-persistent', + options: { path: join(dataDir, 'store.nq') }, + }); + const inspector = createPublisherInspectorFromStore(persistentStore, true); + const jobs = await inspector.publisher.list(); + expect(jobs).toHaveLength(1); + expect(jobs[0]?.jobId).toBeDefined(); + await inspector.stop(); + }); + it('resolves publisher chain defaults from config.networkConfig', async () => { const dataDir = await mkdtemp(join(tmpdir(), 'dkg-publisher-runtime-')); const wallet = ethers.Wallet.createRandom(); diff --git a/packages/cli/test/status-route-rpc.test.ts b/packages/cli/test/status-route-rpc.test.ts index ecc88c7d20..29507ed102 100644 --- a/packages/cli/test/status-route-rpc.test.ts +++ b/packages/cli/test/status-route-rpc.test.ts @@ -33,13 +33,35 @@ import { } from '@origintrail-official/dkg-chain'; import { computeNetworkId } from '../../core/src/genesis.js'; import { getSharedContext } from '../../chain/test/evm-test-context.js'; -import { loadNetworkConfig } from '../src/config.js'; -import { handleStatusRoutes } from '../src/daemon/routes/status.js'; -import type { RequestContext } from '../src/daemon/routes/context.js'; +import { loadNetworkConfig, type DkgConfig } from '../src/config.js'; +import { + handleStatusRoutes, + invalidateExternalStoreQuadsCache, +} from '../src/daemon/routes/status.js'; +import { + createRequestStoreContext, + type RequestContext, +} from '../src/daemon/routes/context.js'; import { startLiveDaemon, stopLiveDaemon, authHeaders, type LiveDaemon } from './helpers/live-daemon.js'; // A port nothing listens on — connecting to it is a REAL refused connection. const DEAD_RPC = 'http://127.0.0.1:9'; +const MANAGED_QUERY_ENDPOINT = 'http://127.0.0.1:7878/query'; + +function managedStoreRuntime(operatorConfig: DkgConfig) { + return { + operatorConfig, + effectiveStore: { backend: 'oxigraph-server', options: {} }, + runtimeStore: { + backend: 'sparql-http', + options: { + queryEndpoint: MANAGED_QUERY_ENDPOINT, + updateEndpoint: 'http://127.0.0.1:7878/update', + managedByDkg: true, + }, + }, + }; +} describe('/api/status + /api/chain/rpc-health (real daemon, real chain)', () => { let daemon: LiveDaemon; @@ -118,9 +140,124 @@ describe('/api/status + /api/chain/rpc-health (real daemon, real chain)', () => }); describe('/api/status selected overlay details', () => { + it('reports the effective managed store for a blockless config', async () => { + const config: DkgConfig = { + name: 'status-blockless-store-test', + nodeRole: 'edge', + chain: { type: 'mock' }, + }; + const query = vi.fn(async () => ({ + type: 'bindings' as const, + bindings: [{ c: '42' }], + })); + invalidateExternalStoreQuadsCache(); + const server = createServer(async (req, res) => { + const url = new URL(req.url ?? '/', 'http://127.0.0.1'); + await handleStatusRoutes({ + req, + res, + path: url.pathname, + url, + network: null, + ...createRequestStoreContext(managedStoreRuntime(config)), + startedAt: Date.now(), + agent: { + peerId: 'peer-status-test', + multiaddrs: [], + node: { + libp2p: { getConnections: () => [] }, + getRelayStats: () => null, + }, + publisher: { getIdentityId: () => 0n }, + store: { query }, + }, + nodeVersion: '0.0.0-test', + nodeCommit: '', + admission: { inFlight: 0, max: 0, rejectedTotal: 0 }, + } as unknown as RequestContext); + }); + + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + try { + const address = server.address() as AddressInfo; + const res = await fetch(`http://127.0.0.1:${address.port}/api/status`); + expect(res.status).toBe(200); + const body: any = await res.json(); + + expect(body.storeBackend).toBe('oxigraph-server'); + expect(body.storeUrl).toBe(MANAGED_QUERY_ENDPOINT); + expect(body.storeQuads).toBe(42); + expect(query).toHaveBeenCalledOnce(); + } finally { + invalidateExternalStoreQuadsCache(); + await new Promise((resolve, reject) => server.close((err) => err ? reject(err) : resolve())); + } + }); + + it('reports the effective managed store after an acknowledged worker cutover', async () => { + const config: DkgConfig = { + name: 'status-worker-cutover-test', + nodeRole: 'edge', + chain: { type: 'mock' }, + store: { backend: 'oxigraph-worker', options: {} }, + }; + const query = vi.fn(async () => ({ + type: 'bindings' as const, + bindings: [{ c: '17' }], + })); + invalidateExternalStoreQuadsCache(); + const server = createServer(async (req, res) => { + const url = new URL(req.url ?? '/', 'http://127.0.0.1'); + await handleStatusRoutes({ + req, + res, + path: url.pathname, + url, + network: null, + ...createRequestStoreContext(managedStoreRuntime(config)), + startedAt: Date.now(), + agent: { + peerId: 'peer-status-test', + multiaddrs: [], + node: { + libp2p: { getConnections: () => [] }, + getRelayStats: () => null, + }, + publisher: { getIdentityId: () => 0n }, + store: { query }, + }, + nodeVersion: '0.0.0-test', + nodeCommit: '', + admission: { inFlight: 0, max: 0, rejectedTotal: 0 }, + } as unknown as RequestContext); + }); + + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + try { + const address = server.address() as AddressInfo; + const res = await fetch(`http://127.0.0.1:${address.port}/api/status`); + expect(res.status).toBe(200); + const body: any = await res.json(); + + expect(body.storeBackend).toBe('oxigraph-server'); + expect(body.storeUrl).toBe(MANAGED_QUERY_ENDPOINT); + expect(body.storeQuads).toBe(17); + expect(query).toHaveBeenCalledOnce(); + } finally { + invalidateExternalStoreQuadsCache(); + await new Promise((resolve, reject) => server.close((err) => err ? reject(err) : resolve())); + } + }); + it('returns the network id and name for the selected overlay genesis', async () => { const network = await loadNetworkConfig('mainnet-gnosis'); expect(network).not.toBeNull(); + const config: DkgConfig = { + name: 'status-selected-overlay-test', + networkConfig: 'mainnet-gnosis', + nodeRole: 'edge', + chain: { type: 'mock' }, + }; const server = createServer(async (req, res) => { const url = new URL(req.url ?? '/', 'http://127.0.0.1'); @@ -130,12 +267,7 @@ describe('/api/status selected overlay details', () => { path: url.pathname, url, network, - config: { - name: 'status-selected-overlay-test', - networkConfig: 'mainnet-gnosis', - nodeRole: 'edge', - chain: { type: 'mock' }, - }, + ...createRequestStoreContext(managedStoreRuntime(config)), startedAt: Date.now(), agent: { peerId: 'peer-status-test', @@ -175,6 +307,17 @@ describe('/api/status selected overlay details', () => { const warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {}); const logSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); const network = await loadNetworkConfig('mainnet-gnosis'); + const config: DkgConfig = { + name: 'status-failover-counter-test', + networkConfig: 'mainnet-gnosis', + nodeRole: 'edge', + chain: { + type: 'evm', + rpcUrl: 'http://127.0.0.1:9', + hubAddress: `0x${'ab'.repeat(20)}`, + chainId: 'evm:31337', + }, + }; try { // Seed the process-wide failover counters the status route reads, then // assert /api/status reflects the exact delta. This would FAIL if the @@ -195,17 +338,7 @@ describe('/api/status selected overlay details', () => { path: url.pathname, url, network, - config: { - name: 'status-failover-counter-test', - networkConfig: 'mainnet-gnosis', - nodeRole: 'edge', - chain: { - type: 'evm', - rpcUrl: 'http://127.0.0.1:9', - hubAddress: `0x${'ab'.repeat(20)}`, - chainId: 'evm:31337', - }, - }, + ...createRequestStoreContext(managedStoreRuntime(config)), startedAt: Date.now(), agent: { peerId: 'peer-status-test', diff --git a/packages/cli/test/store-backend-taxonomy.test.ts b/packages/cli/test/store-backend-taxonomy.test.ts new file mode 100644 index 0000000000..c904cb28c2 --- /dev/null +++ b/packages/cli/test/store-backend-taxonomy.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, it, vi } from 'vitest'; +import { + STORAGE_ADAPTERS, + classifyTripleStoreBackend, + customTripleStoreBackend, + isExternalBackend, + isStorageAdapterBackend, + storageAdapterNames, +} from '@origintrail-official/dkg-storage'; +import { validateStoreConfig, type DkgConfig } from '../src/config.js'; +import { + DEFAULT_DAEMON_STORE_BACKEND, + MANAGED_DAEMON_STORE_BACKEND, + STORE_BACKENDS, + configBackendNames, + isManagedLocalBackend, + isRetiredStoreBackend, + requireStorageAdapterBackend, + storeFlagBackendNames, + storeBackendNames, + wizardBackendChoices, + type StoreBackend, +} from '../src/store-backends.js'; +import { checkExternalStoreReachable } from '../src/daemon/store-health-check.js'; +import { planManagedOxigraph } from '../src/daemon/oxigraph-managed.js'; +import { storeBackendHasStatusHealth } from '../src/daemon/routes/status.js'; + +function configForBackend(backend: StoreBackend): DkgConfig { + const policy = STORE_BACKENDS[backend]; + const options = policy.kind === 'external' + ? { [policy.queryEndpointOption]: 'http://store.test/query' } + : {}; + return { + name: 'taxonomy-test', + apiPort: 9200, + listenPort: 4001, + nodeRole: 'edge', + store: { backend, options }, + } as DkgConfig; +} + +describe('canonical store backend taxonomy', () => { + it('drives config validation and wizard discovery for every registered backend', () => { + const configBackends = configBackendNames(); + const wizardBackends = wizardBackendChoices(); + const flagBackends = storeFlagBackendNames(); + + for (const backend of storeBackendNames()) { + const policy = STORE_BACKENDS[backend]; + const errors = validateStoreConfig(configForBackend(backend)); + + expect((configBackends as readonly StoreBackend[]).includes(backend), backend).toBe(!policy.retired); + expect((wizardBackends as readonly StoreBackend[]).includes(backend), backend).toBe(!policy.retired && policy.wizard); + expect((flagBackends as readonly StoreBackend[]).includes(backend), backend).toBe(!policy.retired && policy.storeFlag); + expect(errors.some((error) => error.field === 'store.backend'), backend).toBe(policy.retired); + if (!policy.retired) expect(errors, backend).toEqual([]); + if (policy.wizard) expect('label' in policy && policy.label.length > 0, backend).toBe(true); + } + }); + + it('keeps daemon health, managed startup, and status routing aligned for every backend', async () => { + for (const backend of storeBackendNames()) { + const policy = STORE_BACKENDS[backend]; + const external = policy.kind === 'external'; + const managed = policy.kind === 'managed-local'; + + expect(isExternalBackend(backend), backend).toBe(external); + expect(isManagedLocalBackend(backend), backend).toBe(managed); + expect(isRetiredStoreBackend(backend), backend).toBe(policy.retired); + expect(isStorageAdapterBackend(backend), backend).toBe(policy.adapter); + expect(classifyTripleStoreBackend(backend).kind, backend).toBe( + policy.adapter ? 'adapter' : 'custom', + ); + expect(storeBackendHasStatusHealth(backend), backend).toBe(external || managed); + expect(planManagedOxigraph(configForBackend(backend), '/data') !== null, backend).toBe(managed); + + const fetch = vi.fn(async () => new Response('{}', { status: 200 })); + const health = await checkExternalStoreReachable({ + storeConfig: configForBackend(backend).store, + fetch, + }); + expect(health.ok, backend).toBe(true); + expect(fetch.mock.calls.length > 0, backend).toBe(external); + } + }); + + it('derives the daemon default and managed-local constant from the registry', () => { + expect(DEFAULT_DAEMON_STORE_BACKEND).toBe(MANAGED_DAEMON_STORE_BACKEND); + expect(STORE_BACKENDS[DEFAULT_DAEMON_STORE_BACKEND]).toMatchObject({ + default: true, + kind: 'managed-local', + retired: false, + }); + }); + + it('composes daemon policy over the storage-owned adapter registry', () => { + expect(storageAdapterNames()).toEqual([ + 'oxigraph', + 'oxigraph-persistent', + 'blazegraph', + 'sparql-http', + ]); + expect(storageAdapterNames()).not.toContain('oxigraph-server'); + expect(storageAdapterNames()).not.toContain('oxigraph-worker'); + for (const backend of storageAdapterNames()) { + expect(STORE_BACKENDS[backend]).toMatchObject(STORAGE_ADAPTERS[backend]); + expect(requireStorageAdapterBackend(backend)).toBe(backend); + } + expect(() => requireStorageAdapterBackend('oxigraph-server')).toThrow( + /not a constructible storage adapter/, + ); + }); + + it('requires an explicit custom-backend escape hatch outside the known registry', () => { + const backend = customTripleStoreBackend('vendor-plugin-store'); + expect(classifyTripleStoreBackend(backend)).toEqual({ + kind: 'custom', + backend: 'vendor-plugin-store', + }); + expect(() => customTripleStoreBackend('oxigraph')).toThrow(/known triple-store adapter/i); + }); +}); diff --git a/packages/cli/test/store-health-check.test.ts b/packages/cli/test/store-health-check.test.ts index b01d80ba5a..49797a2ea8 100644 --- a/packages/cli/test/store-health-check.test.ts +++ b/packages/cli/test/store-health-check.test.ts @@ -37,7 +37,7 @@ describe('checkExternalStoreReachable', () => { it('passes through with no I/O for local backends', async () => { const { fn, calls } = mockFetch(() => new Response(null, { status: 200 })); const result = await checkExternalStoreReachable({ - storeConfig: { backend: 'oxigraph-worker' }, + storeConfig: { backend: 'oxigraph' }, fetch: fn, }); expect(result.ok).toBe(true); diff --git a/packages/cli/test/store-identity-tag.test.ts b/packages/cli/test/store-identity-tag.test.ts index fd8b19d354..f438be62d2 100644 --- a/packages/cli/test/store-identity-tag.test.ts +++ b/packages/cli/test/store-identity-tag.test.ts @@ -43,7 +43,7 @@ describe('checkOrSetStoreIdentity', () => { it('skips for local oxigraph backend', async () => { const { fn, calls } = mockFetch(() => new Response(null, { status: 200 })); const result = await checkOrSetStoreIdentity({ - storeConfig: { backend: 'oxigraph-worker' }, + storeConfig: { backend: 'oxigraph' }, nodeName: 'mynode', fetch: fn, }); diff --git a/packages/cli/test/store-runtime.test.ts b/packages/cli/test/store-runtime.test.ts new file mode 100644 index 0000000000..15fbdab1b9 --- /dev/null +++ b/packages/cli/test/store-runtime.test.ts @@ -0,0 +1,164 @@ +import { afterEach, describe, expect, expectTypeOf, it, vi } from 'vitest'; +import { mkdtemp, readFile, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + resolveDaemonStoreBootPlan, + resolveDaemonStoreRuntime, + type DaemonStoreBootDecision, + type DaemonStoreBootPlan, +} from '../src/daemon/store-runtime.js'; +import { saveConfig, type DkgConfig } from '../src/config.js'; +import type { StorageAdapterBackend } from '@origintrail-official/dkg-storage'; + +function mk(overrides: Partial = {}): DkgConfig { + return { + name: 'dkg-node', + apiPort: 9200, + listenPort: 4001, + nodeRole: 'edge', + ...overrides, + } as DkgConfig; +} + +function expectBootable( + decision: DaemonStoreBootDecision, +): asserts decision is DaemonStoreBootPlan { + expect(decision.kind).toBe('bootable'); + if (decision.kind !== 'bootable') { + throw new Error(`Expected a bootable store plan, got ${decision.kind}`); + } +} + +describe('resolveDaemonStoreBootPlan', () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it('keeps blockless operator config separate from the materialized daemon store default', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-')); + const config = mk(); + + const plan = resolveDaemonStoreBootPlan({ + config, + dataDir, + acceptStoreReset: false, + }); + + expectBootable(plan); + expect(plan.operatorConfig).toBe(config); + expect(plan.operatorConfig.store).toBeUndefined(); + expect(plan.effectiveConfig.store).toEqual({ backend: 'oxigraph-server', options: {} }); + expect(plan.effectiveStore).toEqual({ backend: 'oxigraph-server', options: {} }); + }); + + it('fails at runtime-plan construction when a managed store was not materialized', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-unmaterialized-')); + const plan = resolveDaemonStoreBootPlan({ + config: mk(), + dataDir, + acceptStoreReset: false, + }); + + expectBootable(plan); + expect(() => resolveDaemonStoreRuntime(plan, null)).toThrow( + /oxigraph-server.*not materialized to a storage adapter/, + ); + }); + + it('refines a live runtime store to a constructible adapter backend', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-adapter-')); + const plan = resolveDaemonStoreBootPlan({ + config: mk({ store: { backend: 'oxigraph', options: {} } }), + dataDir, + acceptStoreReset: false, + }); + + expectBootable(plan); + const runtime = resolveDaemonStoreRuntime(plan, null); + expect(runtime.runtimeStore.backend).toBe('oxigraph'); + expectTypeOf(runtime.runtimeStore.backend).toMatchTypeOf(); + }); + + it('does not persist the materialized store default during an unrelated config save', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-save-')); + vi.stubEnv('DKG_HOME', dataDir); + const plan = resolveDaemonStoreBootPlan({ + config: mk(), + dataDir, + acceptStoreReset: false, + }); + + expectBootable(plan); + plan.operatorConfig.sharedMemoryTtlMs = 1234; + await saveConfig(plan.operatorConfig); + + const persisted = JSON.parse(await readFile(join(dataDir, 'config.json'), 'utf8')) as DkgConfig; + expect(persisted.sharedMemoryTtlMs).toBe(1234); + expect(persisted.store).toBeUndefined(); + expect(plan.effectiveConfig.store).toEqual({ backend: 'oxigraph-server', options: {} }); + }); + + it('gates an oxigraph-server cutover when legacy store.nq exists without a backend marker', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-cutover-')); + await writeFile(join(dataDir, 'store.nq'), '

.'); + + const plan = resolveDaemonStoreBootPlan({ + config: mk({ store: { backend: 'oxigraph-server', options: {} } }), + dataDir, + acceptStoreReset: false, + }); + + expect(plan.kind).toBe('blocked-legacy-cutover'); + if (plan.kind !== 'blocked-legacy-cutover') { + throw new Error(`Expected a blocked legacy cutover, got ${plan.kind}`); + } + expect(plan.message).toContain('DKG_ACCEPT_STORE_RESET=1'); + }); + + it('does not repeatedly gate a cutover already recorded as oxigraph-server', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-recorded-')); + await writeFile(join(dataDir, 'store.nq'), '

.'); + await writeFile( + join(dataDir, '.network-state.json'), + JSON.stringify({ chainResetMarker: null, lastBackend: 'oxigraph-server', savedAt: Date.now() }), + ); + + const plan = resolveDaemonStoreBootPlan({ + config: mk(), + dataDir, + acceptStoreReset: false, + }); + + expectBootable(plan); + expect(plan.effectiveStore.backend).toBe('oxigraph-server'); + }); + + it('migrates an explicit worker config only after acknowledging its legacy store', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-worker-')); + await writeFile(join(dataDir, 'store.nq'), '

.'); + const config = mk({ store: { backend: 'oxigraph-worker' } }); + + const blocked = resolveDaemonStoreBootPlan({ config, dataDir, acceptStoreReset: false }); + expect(blocked.kind).toBe('blocked-legacy-cutover'); + if (blocked.kind !== 'blocked-legacy-cutover') { + throw new Error(`Expected a blocked legacy cutover, got ${blocked.kind}`); + } + expect(blocked.message).toContain('legacy store.nq from the old oxigraph-worker backend'); + + const acknowledged = resolveDaemonStoreBootPlan({ config, dataDir, acceptStoreReset: true }); + expectBootable(acknowledged); + expect(acknowledged.effectiveStore).toEqual({ backend: 'oxigraph-server', options: {} }); + expect(acknowledged.operatorConfig).toBe(config); + expect(acknowledged.operatorConfig.store?.backend).toBe('oxigraph-worker'); + }); + + it('classifies an explicit worker config without legacy data as invalid', async () => { + const dataDir = await mkdtemp(join(tmpdir(), 'dkg-store-runtime-invalid-worker-')); + const config = mk({ store: { backend: 'oxigraph-worker' } }); + + const decision = resolveDaemonStoreBootPlan({ config, dataDir, acceptStoreReset: false }); + + expect(decision).toEqual({ kind: 'invalid-config', operatorConfig: config }); + }); +}); diff --git a/packages/cli/test/store-wizard.test.ts b/packages/cli/test/store-wizard.test.ts index 02a2ac794a..a67557bba3 100644 --- a/packages/cli/test/store-wizard.test.ts +++ b/packages/cli/test/store-wizard.test.ts @@ -10,7 +10,7 @@ * - Blazegraph + valid URL: store block persisted with * `managedByDkg: false`. * - Blazegraph + unreachable URL: surfaces formatted failure, allows - * retry, abort returns to local default. + * retry, abort leaves the managed local default in place. * - Blazegraph + 404 URL: namespace-missing branch fires; message * mentions namespace, not network. * - Blank URL prompt: PR 2's "no Docker yet" message + retry. @@ -28,6 +28,14 @@ import { describe, it, expect } from 'vitest'; import { applyStoreFlagsToConfig, promptStoreBackend } from '../src/store-wizard.js'; import type { DkgConfig } from '../src/config.js'; +import { + STORE_BACKENDS, + configBackendList, + configBackendNames, + storeFlagBackendList, + storeFlagBackendNames, + wizardBackendChoices, +} from '../src/store-backends.js'; function mockFetch(handler: (input: any, init?: any) => Response | Promise) { const calls: Array<{ url: string; init?: RequestInit }> = []; @@ -54,6 +62,26 @@ function mockAsk(scriptedAnswers: string[]): (q: string, def?: string) => Promis // --------------------------------------------------------------------- describe('promptStoreBackend', () => { + it('keeps config, wizard, and flag backend lists aligned with their policies', () => { + expect(configBackendList().split(', ')).toEqual(configBackendNames()); + expect(configBackendNames()).toEqual([ + 'oxigraph-server', + 'oxigraph', + 'oxigraph-persistent', + 'blazegraph', + 'sparql-http', + ]); + expect(storeFlagBackendList().split(', ')).toEqual(storeFlagBackendNames()); + expect(storeFlagBackendNames()).toEqual([ + 'oxigraph-server', + 'oxigraph', + 'blazegraph', + 'sparql-http', + ]); + expect(wizardBackendChoices()).toEqual(['oxigraph-server', 'oxigraph', 'blazegraph']); + expect(Object.keys(STORE_BACKENDS)).toContain('oxigraph-worker'); + }); + it('defaults to oxigraph-server when the operator accepts the default', async () => { const { fn, calls } = mockFetch(() => new Response(null, { status: 200 })); const result = await promptStoreBackend({ @@ -65,31 +93,26 @@ describe('promptStoreBackend', () => { expect(calls).toHaveLength(0); // no URL probe issued for a local backend }); - it('returns no store block (embedded worker) when operator picks "oxigraph" by name', async () => { + it('persists an explicit embedded development store when operator picks "oxigraph" by name', async () => { const result = await promptStoreBackend({ ask: mockAsk(['oxigraph']), log: () => {}, }); - expect(result.storeBlock).toBeNull(); + expect(result.storeBlock).toEqual({ backend: 'oxigraph', options: {} }); }); - it('returns no store block (embedded worker) when operator picks the worker by number', async () => { + it('persists an explicit embedded development store when operator picks oxigraph by number', async () => { // Menu is now `1) oxigraph-server 2) oxigraph 3) blazegraph` — picking - // option 2 must opt down to the embedded in-process worker (no block). + // option 2 opts into the embedded in-memory store explicitly. const result = await promptStoreBackend({ ask: mockAsk(['2']), log: () => {}, }); - expect(result.storeBlock).toBeNull(); + expect(result.storeBlock).toEqual({ backend: 'oxigraph', options: {} }); }); - it('preserves an explicit embedded backend verbatim on Enter-through (no flip, no option loss)', async () => { - // Codex #946 — only a *block-less* config should fall through to the new - // oxigraph-server default. A node that explicitly chose a local worker - // variant must keep it on a re-init Enter-through, AND keep its custom - // `options` (e.g. the worker's `options.path`): returning `null` would let - // `dkg init` write `store: undefined` and relocate the store on next boot. - for (const backend of ['oxigraph', 'oxigraph-worker', 'oxigraph-persistent'] as const) { + it('preserves a supported explicit embedded backend verbatim on Enter-through (no flip, no option loss)', async () => { + for (const backend of ['oxigraph', 'oxigraph-persistent'] as const) { const existingStore = { backend, options: { path: '/custom/store' } }; const result = await promptStoreBackend({ ask: mockAsk(['']), // Enter @@ -100,11 +123,7 @@ describe('promptStoreBackend', () => { } }); - it('switches to the default embedded worker when an oxigraph-persistent node EXPLICITLY picks oxigraph', async () => { - // Codex #946 — preservation must be gated on a true keep. An operator who - // explicitly selects option `2` / "oxigraph" to move a worker/persistent - // node back to the plain embedded default must NOT have the old backend + - // options silently retained. Both the numeric and named selection switch. + it('does not preserve oxigraph-persistent options when the operator explicitly picks oxigraph', async () => { const existingStore = { backend: 'oxigraph-persistent', options: { path: '/custom/store' } }; for (const answer of ['2', 'oxigraph']) { const result = await promptStoreBackend({ @@ -112,13 +131,33 @@ describe('promptStoreBackend', () => { existingStore, log: () => {}, }); - expect(result.storeBlock).toBeNull(); + expect(result.storeBlock).toEqual({ backend: 'oxigraph', options: {} }); } }); + it('does not preserve retired oxigraph-worker configs on Enter-through', async () => { + const logs: string[] = []; + const result = await promptStoreBackend({ + ask: mockAsk(['']), + existingStore: { backend: 'oxigraph-worker', options: { path: '/custom/store' } }, + log: (m) => logs.push(m), + }); + expect(result.storeBlock).toEqual({ backend: 'oxigraph-server', options: {} }); + expect(logs.join('\n')).toMatch(/retired/); + }); + + it('rejects oxigraph-worker when typed explicitly', async () => { + await expect( + promptStoreBackend({ + ask: mockAsk(['oxigraph-worker']), + log: () => {}, + }), + ).rejects.toThrow(/no longer supported/); + }); + it('falls back to the recommended default (oxigraph-server) on an out-of-range number', async () => { // Codex #946 — a typo'd digit ("9") must not silently downgrade a fresh - // install to the embedded worker; it resolves to defaultBackend (option 1). + // install to an embedded backend; it resolves to defaultBackend (option 1). const result = await promptStoreBackend({ ask: mockAsk(['9']), log: () => {}, @@ -172,7 +211,7 @@ describe('promptStoreBackend', () => { expect(logs.some((l) => l.includes('STORE-HEALTH'))).toBe(true); }); - it('aborts to local default when operator declines retry on unreachable URL', async () => { + it('aborts to the managed local default when operator declines retry on unreachable URL', async () => { const { fn } = mockFetch(() => new Response('boom', { status: 500 })); const logs: string[] = []; const result = await promptStoreBackend({ @@ -283,7 +322,7 @@ describe('promptStoreBackend', () => { 'blazegraph', '', // blank URL 'n', // decline Docker - 'n', // decline retry-with-URL → abort to local default + 'n', // decline retry-with-URL → abort to managed local default ]), isDockerAvailable: async () => true, provisionBlazegraphDocker: async () => { @@ -616,7 +655,22 @@ describe('applyStoreFlagsToConfig', () => { storeFlag: 'neptune', log: () => {}, }), - ).rejects.toThrow(/oxigraph, blazegraph, sparql-http/); + ).rejects.toThrow(/oxigraph-server, oxigraph, blazegraph, sparql-http/); + }); + + it('does not advertise or accept oxigraph-persistent as a pathless --store choice', async () => { + const store = newMockConfig({ + ...baseConfig, + store: { backend: 'oxigraph-persistent', options: { path: '/existing/store.nq' } }, + } as DkgConfig); + const io = mockConfigIO(store); + + await expect(applyStoreFlagsToConfig({ + ...io, + storeFlag: 'oxigraph-persistent', + log: () => {}, + })).rejects.toThrow(/--store must be one of: oxigraph-server, oxigraph, blazegraph, sparql-http/); + expect(store.saved).toEqual([]); }); it('persists a daemon-managed oxigraph-server block (no URL required)', async () => { @@ -654,7 +708,7 @@ describe('applyStoreFlagsToConfig', () => { expect(store.saved[0].store).toEqual({ backend: 'oxigraph-server', options: {} }); }); - it('clears existing store block when --store oxigraph is passed', async () => { + it('persists an explicit oxigraph block when --store oxigraph is passed', async () => { const store = newMockConfig({ ...baseConfig, store: { @@ -669,10 +723,10 @@ describe('applyStoreFlagsToConfig', () => { log: () => {}, }); expect(store.saved).toHaveLength(1); - expect(store.saved[0].store).toBeUndefined(); + expect(store.saved[0].store).toEqual({ backend: 'oxigraph', options: {} }); }); - it('is a no-op when --store oxigraph is passed and no existing store block', async () => { + it('persists oxigraph when --store oxigraph is passed and no existing store block', async () => { const store = newMockConfig(baseConfig); const io = mockConfigIO(store); await applyStoreFlagsToConfig({ @@ -680,6 +734,20 @@ describe('applyStoreFlagsToConfig', () => { storeFlag: 'oxigraph', log: () => {}, }); + expect(store.saved).toHaveLength(1); + expect(store.saved[0].store).toEqual({ backend: 'oxigraph', options: {} }); + }); + + it('rejects --store oxigraph-worker', async () => { + const store = newMockConfig(baseConfig); + const io = mockConfigIO(store); + await expect( + applyStoreFlagsToConfig({ + ...io, + storeFlag: 'oxigraph-worker', + log: () => {}, + }), + ).rejects.toThrow(/no longer supported/); expect(store.saved).toEqual([]); }); diff --git a/packages/cli/test/validate-store-config.test.ts b/packages/cli/test/validate-store-config.test.ts index 39ff79e24c..fefd180a61 100644 --- a/packages/cli/test/validate-store-config.test.ts +++ b/packages/cli/test/validate-store-config.test.ts @@ -8,8 +8,8 @@ * when paired with an external backend (no local store path to * infer from). * - * Local backends (default Oxigraph) are unaffected — the function is a - * no-op for them. + * Supported local backends are unaffected. The retired `oxigraph-worker` + * backend is rejected before boot. * * Plan: `.cursor/plans/blazegraph_v10_support_178da670.plan.md` §PR 1 item 6. */ @@ -33,9 +33,12 @@ describe('validateStoreConfig', () => { }); describe('local backends', () => { - it('no-op for oxigraph-worker', () => { + it('no-op for supported local backends', () => { expect( - validateStoreConfig(mk({ store: { backend: 'oxigraph-worker' } })), + validateStoreConfig(mk({ store: { backend: 'oxigraph' } })), + ).toEqual([]); + expect( + validateStoreConfig(mk({ store: { backend: 'oxigraph-persistent', options: { path: '/tmp/store.nq' } } })), ).toEqual([]); }); @@ -44,10 +47,19 @@ describe('validateStoreConfig', () => { // if the backend is local; the wipe + health check honour // isExternalBackend the same way. const errors = validateStoreConfig( - mk({ store: { backend: 'oxigraph-worker', options: { url: 'irrelevant' } } }), + mk({ store: { backend: 'oxigraph', options: { url: 'irrelevant' } } }), ); expect(errors).toEqual([]); }); + + it('rejects the retired oxigraph-worker backend', () => { + const errors = validateStoreConfig( + mk({ store: { backend: 'oxigraph-worker' } }), + ); + expect(errors).toHaveLength(1); + expect(errors[0].field).toBe('store.backend'); + expect(errors[0].message).toMatch(/no longer supported/); + }); }); describe('blazegraph', () => { @@ -136,7 +148,7 @@ describe('validateStoreConfig', () => { it('does not enforce the directory requirement for local backends', () => { const errors = validateStoreConfig( mk({ - store: { backend: 'oxigraph-worker' }, + store: { backend: 'oxigraph-persistent', options: { path: '/tmp/store.nq' } }, largeLiteralStorage: { enabled: true }, }), ); diff --git a/packages/cli/test/write-preflight-resilience.test.ts b/packages/cli/test/write-preflight-resilience.test.ts index 3428effdca..2e3c44dca4 100644 --- a/packages/cli/test/write-preflight-resilience.test.ts +++ b/packages/cli/test/write-preflight-resilience.test.ts @@ -12,9 +12,9 @@ // `contextGraphActivePublicOnChainFromRegistry` / `contextGraphExists` // methods (invoked via ContextGraphResolveMethods.prototype on a narrow // harness carrying real state, the same shape the DKGAgent mixin sees), and -// • a REAL OxigraphWorkerStore that has been close()d — the honest -// "the store is closed" failure a crashed/closed worker produces live — -// alongside a healthy in-memory worker store for the no-change paths, and +// • a REAL SparqlHttpStore pointed at an unavailable loopback endpoint — the +// honest connection failure an unavailable external/managed store produces +// live — alongside a healthy embedded store for the no-change paths, and // • the REAL `resolveRequiredWriteContextGraphId` resolver with a real // captured ServerResponse sink (same conventions as // context-graph-write-path-validation.test.ts). @@ -38,7 +38,7 @@ import { WRITE_PREFLIGHT_CHAIN_RESCUE_TIMEOUT_MS, } from '../src/daemon/http-utils.js'; import { ContextGraphResolveMethods } from '../../agent/src/dkg-agent-cg-resolve.js'; -import { OxigraphWorkerStore, createTripleStore, type TripleStore } from '@origintrail-official/dkg-storage'; +import { createTripleStore, type TripleStore } from '@origintrail-official/dkg-storage'; import { DKG_ONTOLOGY, SYSTEM_CONTEXT_GRAPHS, @@ -104,8 +104,8 @@ function agentHarness( // The narrow data interface resolveRequiredWriteContextGraphId consumes. // `listContextGraphs` performs a real read against the harness store so a -// closed store rejects with the genuine worker error (the same failure mode -// the real list path hits), and a healthy store returns the given rows. +// unavailable store rejects with a genuine adapter error (the same failure +// mode the real list path hits), and a healthy store returns the given rows. function providerFor(harness: any, rows: Array> = []) { const listCalls: Array = []; return { @@ -153,25 +153,30 @@ function captureRes(): { res: ServerResponse; out: { status?: number; body?: any // The exact legacy 503 body the pre-resilience code produced when both legs // threw. Live callers and tests grep for this — the rescue must leave it // byte-compatible whenever it does NOT accept. -const LEGACY_503_ERROR = /^Failed to validate contextGraphId against known context graphs: exact preflight failed: .*store is closed.*; list validation failed: .*store is closed/; +const LEGACY_503_ERROR = /^Failed to validate contextGraphId against known context graphs: exact preflight failed: .*fetch failed.*; list validation failed: .*fetch failed/; -let closedStore: OxigraphWorkerStore; -let healthyStore: OxigraphWorkerStore; +let closedStore: TripleStore; +let healthyStore: TripleStore; beforeAll(async () => { - // A real worker-backed store that has been closed: every subsequent call - // rejects with the real `oxigraph-worker: cannot run "…" — the store is - // closed.` error — the exact live failure this track fixes. - closedStore = new OxigraphWorkerStore(undefined); - await closedStore.close(); - healthyStore = new OxigraphWorkerStore(undefined); + // A real external-store adapter pointed at an unavailable endpoint: every + // query rejects through the same fetch path used by managed/external stores. + closedStore = await createTripleStore({ + backend: 'sparql-http', + options: { + queryEndpoint: 'http://127.0.0.1:65535/query', + updateEndpoint: 'http://127.0.0.1:65535/update', + }, + }); + healthyStore = await createTripleStore({ backend: 'oxigraph' }); }); afterAll(async () => { + await closedStore.close(); await healthyStore.close(); }); -describe('probeContextGraphWritePreflight — store-failure resilience (real probe, real closed store)', () => { +describe('probeContextGraphWritePreflight — store-failure resilience (real probe, unavailable store)', () => { it('detects local content from indexed graph names while ignoring bookkeeping-only graphs', async () => { const store = await createTripleStore({ backend: 'oxigraph' }); const listGraphsByPrefix = store.listGraphsByPrefix?.bind(store); @@ -230,7 +235,7 @@ describe('probeContextGraphWritePreflight — store-failure resilience (real pro ); const probe = await harness.probeContextGraphWritePreflight(CG); expect(probe.storeUnavailable).toBe(true); - expect(probe.storeErrorMessage).toMatch(/store is closed/); + expect(probe.storeErrorMessage).toMatch(/fetch failed/); // In-memory registry state needs zero store I/O and must survive. expect(probe.inMemorySubscription).toEqual({ subscribed: true, synced: true }); // Store-derived facts are UNKNOWN — a store outage must never be @@ -269,7 +274,7 @@ describe('probeContextGraphWritePreflight — store-failure resilience (real pro }); }); -describe('resolveRequiredWriteContextGraphId — both-legs-failed rescue (real resolver, real closed store)', () => { +describe('resolveRequiredWriteContextGraphId — both-legs-failed rescue (real resolver, unavailable store)', () => { it('(a) ACCEPTS on positive on-chain proof the CG is active AND PUBLIC', async () => { const isActive = recorder(async (id: bigint) => id === 7n); const getAccessPolicy = recorder(async (_id: bigint) => 0); // 0 = public diff --git a/packages/cli/vitest.unit.config.ts b/packages/cli/vitest.unit.config.ts index 9fbdf64fbf..3372b91cd1 100644 --- a/packages/cli/vitest.unit.config.ts +++ b/packages/cli/vitest.unit.config.ts @@ -87,8 +87,8 @@ export default defineConfig({ // #761 — context graph write-target validation (from main). 'test/context-graph-write-path-validation.test.ts', // Track B — write-preflight resilience when the local store is - // slow/closed. Real resolver + real agent probe + real (closed) - // OxigraphWorkerStore; no hardhat needed. + // slow/unavailable. Real resolver + real agent probe + real + // unavailable SPARQL adapter; no hardhat needed. 'test/write-preflight-resilience.test.ts', 'test/http-literal-size-validation.test.ts', // CLI subprocess smoke with stub daemon only; no hardhat needed. diff --git a/packages/core/src/proto/storage-ack.ts b/packages/core/src/proto/storage-ack.ts index 11b2fafc61..d22b8d9e92 100644 --- a/packages/core/src/proto/storage-ack.ts +++ b/packages/core/src/proto/storage-ack.ts @@ -111,8 +111,8 @@ export const STORAGE_ACK_DECLINE_CODES = { /** * The core hit a peer-LOCAL, transient infrastructure failure while * servicing an otherwise well-formed request: its triple store errored - * mid-read/write (e.g. an oxigraph worker mid-restart throwing - * `store is closed`) or its live signer-registration chain lookup threw + * mid-read/write (e.g. a managed or external store restarting and refusing + * a connection) or its live signer-registration chain lookup threw * (a degraded shared RPC). Introduced after the testnet storage-ACK * dead-air incident: every such failure previously THREW out of the * handler, which ProtocolRouter's inbound wrapper surfaces as a bare @@ -151,7 +151,7 @@ export const TRANSIENT_STORAGE_ACK_DECLINE_CODES: ReadonlySet = new Set< // waiting fixes it. STORAGE_ACK_DECLINE_CODES.MISSING_CIPHERTEXT_CHUNKS, // Dead-air fix: a store/RPC blip on the core clears when the local - // oxigraph worker or the shared RPC recovers — the same "wait a few + // local store or the shared RPC recovers — the same "wait a few // seconds and re-ask" cadence as SWM catch-up. Marking it transient // keeps a briefly-degraded core in the quorum pool instead of // deselecting it on the first blip. diff --git a/packages/core/test/ensure-dkg-node-config.test.ts b/packages/core/test/ensure-dkg-node-config.test.ts index 6c2d519fea..8fc412231c 100644 --- a/packages/core/test/ensure-dkg-node-config.test.ts +++ b/packages/core/test/ensure-dkg-node-config.test.ts @@ -71,7 +71,7 @@ describe('ensureDkgNodeConfig — store-backend default (issue #960)', () => { it('does NOT flip an existing (block-less) node onto a new backend', () => { // Simulate an existing node: a config.json is already on disk (it had been - // running on the oxigraph-worker runtime fallback). Re-running setup must + // running without an explicit store block). Re-running setup must // not silently switch its backend (which would force a store reset). writeFileSync(join(tempHome, 'config.json'), JSON.stringify({ name: 'node-a', nodeRole: 'edge' }) + '\n'); ensureDkgNodeConfig({ diff --git a/packages/kafka-plugin/test/kafka-plugin-api.e2e.test.ts b/packages/kafka-plugin/test/kafka-plugin-api.e2e.test.ts index dd08785081..3abebdcd3f 100644 --- a/packages/kafka-plugin/test/kafka-plugin-api.e2e.test.ts +++ b/packages/kafka-plugin/test/kafka-plugin-api.e2e.test.ts @@ -124,7 +124,7 @@ async function writeDaemonConfig( relay: 'none', auth: { enabled: true }, store: { - backend: 'oxigraph-worker', + backend: 'oxigraph-persistent', options: { path: join(home, 'store.nq') }, }, chain: { diff --git a/packages/node-ui/e2e/specs/devnet/publish-ack-quorum.devnet.spec.ts b/packages/node-ui/e2e/specs/devnet/publish-ack-quorum.devnet.spec.ts index c057c13d7c..3732dc9d76 100644 --- a/packages/node-ui/e2e/specs/devnet/publish-ack-quorum.devnet.spec.ts +++ b/packages/node-ui/e2e/specs/devnet/publish-ack-quorum.devnet.spec.ts @@ -17,7 +17,7 @@ * slow-RPC fault legs those PRs also fixed cannot be induced against a healthy * black-box devnet — they are pinned by the unit/integration suites shipped in * #1404 (publisher-runner-ack-readiness, policy-retry) and #1408 - * (storage-ack-core-unavailable, oxigraph-worker-respawn). + * (storage-ack-core-unavailable and related store-outage scenarios). */ import { test, expect } from '../../fixtures/base.js'; import { requireDevnetNode, requireDevnetPrecondition, waitForDevnetStatus } from '../../helpers/devnet.js'; diff --git a/packages/node-ui/e2e/specs/devnet/write-preflight-guard.devnet.spec.ts b/packages/node-ui/e2e/specs/devnet/write-preflight-guard.devnet.spec.ts index 487cdaaac2..0eb25f50bb 100644 --- a/packages/node-ui/e2e/specs/devnet/write-preflight-guard.devnet.spec.ts +++ b/packages/node-ui/e2e/specs/devnet/write-preflight-guard.devnet.spec.ts @@ -10,9 +10,9 @@ * node does not track is NEVER admitted; refusals are structured, side-effect * free, and leave the node healthy" — is exactly what this spec pins against * the healthy devnet. (The outage-only legs — 503 fail-closed, the on-chain - * public rescue itself, oxigraph worker respawn — need a killed store and are + * public rescue itself and store restart recovery — need a killed store and are * pinned by #1408's real-component suites: write-preflight-resilience, - * storage-ack-core-unavailable, oxigraph-worker-respawn.) + * storage-ack-core-unavailable and related store-outage scenarios.) */ import { test, expect } from '../../fixtures/base.js'; import { devnetApiFetch, requireDevnetNode, requireDevnetPrecondition, waitForDevnetStatus } from '../../helpers/devnet.js'; diff --git a/packages/query/src/query-handler.ts b/packages/query/src/query-handler.ts index 17fd97c2c0..4bc94d6a41 100644 --- a/packages/query/src/query-handler.ts +++ b/packages/query/src/query-handler.ts @@ -405,7 +405,7 @@ export class QueryHandler { // post-materialization `.slice()`, so a high-cardinality query is fully // materialized before being truncated. Genuinely enforcing either bound // requires threading an `AbortSignal` + result cap through - // `QueryEngine.query()` → storage → the oxigraph worker. Injecting a + // `QueryEngine.query()` → the selected storage adapter. Injecting a // `LIMIT` into the user query here is NOT a safe shortcut: for scoped // queries it would push the statement into the multi-graph // solution-set-modifier rejection path (see #789), so it is diff --git a/packages/storage/README.md b/packages/storage/README.md index 698e3c2a05..08c2263bf4 100644 --- a/packages/storage/README.md +++ b/packages/storage/README.md @@ -6,7 +6,6 @@ Triple store abstraction layer for DKG V10. Provides a unified API over multiple - **Backend adapters** — pluggable triple store implementations: - `OxigraphStore` — embedded WASM/native store, no external dependencies - - `OxigraphWorkerStore` — worker-thread variant; keeps the daemon event loop free, with a per-read-operation timeout (see below) - `BlazegraphStore` — connects to a running Blazegraph SPARQL endpoint - `SparqlHttpStore` — generic adapter for any SPARQL 1.1 compliant endpoint - **Graph manager** — named graph lifecycle (create, drop, list) with contextGraph-scoped data and metadata graphs @@ -33,39 +32,6 @@ await store.insert(quads); const result = await store.query('SELECT * WHERE { ?s ?p ?o } LIMIT 10'); ``` -## Embedded worker store (`oxigraph-worker`) tuning - -The embedded worker runs **all** store operations on a single worker thread, so -a long-running or stuck op (a huge import, an expensive query) blocks every -other store-backed request behind it. Under real load this surfaces as the -daemon's `/api/status` staying green while `/api/query`, -`/api/context-graph/list`, and `/api/assertion/create` hang. A `store.options` -knob bounds that blast radius: - -| Option | Default | Purpose | -|---|---|---| -| `operationTimeoutMs` | `120000` | Reject a **read-only** op (`query`, `hasGraph`, `listGraphs`, `countQuads`) that exceeds this instead of hanging forever — that's where the user-visible hang shows up. `0` disables (restores unbounded behaviour). `close` is exempt — its final flush always runs to completion so shutdown can't drop pending writes. | - -Mutations (`insert`, `delete`, …) are intentionally **not** bounded by this -timeout. The bound only drops the *caller's* promise — the single worker thread -keeps running the op — so a "timed-out" write could still commit afterwards, -and the rest of the codebase treats a rejected `insert`/`delete` as a clean -failure. Bounding only reads surfaces a wedged worker on the paths that hang -without inventing an indeterminate write outcome. `insert()` therefore stays -strictly atomic (all quads commit or the call fails), which callers rely on. - -```jsonc -// ~/.dkg/config.json -"store": { - "backend": "oxigraph-worker", - "options": { "operationTimeoutMs": 120000 } -} -``` - -For heavy / production workloads, prefer an out-of-process SPARQL server -(`sparql-http` or `blazegraph`), which handles reads and writes concurrently -and keeps the daemon responsive under load. - ## Internal Dependencies - `@origintrail-official/dkg-core` — configuration types, logging, constants diff --git a/packages/storage/src/adapters/oxigraph-worker-impl.ts b/packages/storage/src/adapters/oxigraph-worker-impl.ts deleted file mode 100644 index 6bfce261ce..0000000000 --- a/packages/storage/src/adapters/oxigraph-worker-impl.ts +++ /dev/null @@ -1,18 +0,0 @@ -import { parentPort, workerData } from 'node:worker_threads'; -import { OxigraphStore } from './oxigraph.js'; - -const store = new OxigraphStore(workerData?.persistPath); - -parentPort!.on('message', async (msg: { id: number; method: string; args: unknown[] }) => { - try { - const fn = (store as any)[msg.method]; - if (typeof fn !== 'function') { - parentPort!.postMessage({ id: msg.id, error: `Unknown method: ${msg.method}` }); - return; - } - const result = await fn.apply(store, msg.args); - parentPort!.postMessage({ id: msg.id, result }); - } catch (err) { - parentPort!.postMessage({ id: msg.id, error: err instanceof Error ? err.message : String(err) }); - } -}); diff --git a/packages/storage/src/adapters/oxigraph-worker.ts b/packages/storage/src/adapters/oxigraph-worker.ts deleted file mode 100644 index 110866bc84..0000000000 --- a/packages/storage/src/adapters/oxigraph-worker.ts +++ /dev/null @@ -1,737 +0,0 @@ -import { Worker } from 'node:worker_threads'; -import { existsSync } from 'node:fs'; -import { sep } from 'node:path'; -import { fileURLToPath } from 'node:url'; -import type { TripleStore, Quad, TripleStoreQueryOptions, QueryResult, UpdateOptions } from '../triple-store.js'; -import { registerTripleStoreAdapter } from '../triple-store.js'; -import { GraphWriteGenTracker } from '../graph-write-gen.js'; - -/** - * Default per-operation timeout for the embedded worker store. The worker is - * a SINGLE thread that processes store ops FIFO, so one slow / wedged op (a - * huge import, an expensive query, or a genuinely hung worker) blocks every - * other store-backed request queued behind it. Without a bound, the caller — - * an API route, the publisher, gossip ingest — waits FOREVER. That is the - * exact signature behind issues #997 / #999 / #1002 / #1005 / #1008: - * `/api/status` (no store) stays green while `/api/query`, - * `/api/context-graph/list`, `/api/assertion/create` never return. - * - * A bounded wait turns an indefinite hang into a surfaced error the operator - * can act on (the message points at the real fix: use an external SPARQL - * server for heavy workloads). 120s is generous enough not to trip normal - * operations yet finite. Set `operationTimeoutMs: 0` to restore the old - * unbounded behaviour. - * - * IMPORTANT — the timeout is applied to READ-ONLY ops only (see - * READ_ONLY_METHODS). A read is side-effect-free, so bounding the caller's wait - * and rejecting is always a clean, determinate failure. A mutation is NOT - * bounded: the timeout only drops the caller's promise while the single worker - * thread keeps running the op, so a "timed-out" insert/delete could STILL - * commit afterwards — and the rest of the codebase treats a rejected - * insert/delete as a clean failure, which would leave partial state visible and - * retries ambiguous. The user-visible hang in the issues above is on the read - * paths (`/api/query`, `/api/context-graph/list`); bounding reads surfaces a - * wedged worker there without inventing an indeterminate mutation outcome. - */ -const DEFAULT_OPERATION_TIMEOUT_MS = 120_000; - -/** - * Backoff (ms) before each consecutive respawn attempt after an UNEXPECTED - * worker exit: the first attempt is immediate (a one-off OOM/crash should - * recover with zero visible downtime), then 1s / 5s / 30s, capped at the last - * tier. The cap keeps a persistently-crashing worker (corrupt state, chronic - * OOM on load) from melting the node in a hot spawn/crash loop while still - * retrying often enough that a transient cause self-heals. - */ -const RESPAWN_BACKOFF_MS = [0, 1_000, 5_000, 30_000]; - -/** - * Give-up bound: after this many CONSECUTIVE respawned workers die without - * serving a single successful op, stop respawning and latch the store closed - * (the pre-recovery behaviour) with fatal operator guidance. The counter - * resets on the first successful reply from a worker (see the message handler - * in spawnWorker), so occasional crashes days apart never accumulate here — - * only a genuine crash loop trips it. - */ -const MAX_CONSECUTIVE_RESPAWNS = 5; - -/** Unref'd sleep — a respawn backoff timer must not keep the process alive on its own. */ -function sleep(ms: number): Promise { - return new Promise((resolve) => { - const t = setTimeout(resolve, ms); - if (typeof t.unref === 'function') t.unref(); - }); -} - -export interface OxigraphWorkerStoreOptions { - /** - * Per-operation timeout in milliseconds for READ-ONLY ops. Default 120_000. - * 0 disables it. Mutations are intentionally never bounded (see - * DEFAULT_OPERATION_TIMEOUT_MS) so a timed-out write can't be reported as a - * clean failure while it is still in flight. - */ - operationTimeoutMs?: number; -} - -/** - * Accept only a finite, non-negative override; otherwise fall back. The result - * is floored to an INTEGER — the timeout is a millisecond count, so a fractional - * value is meaningless noise. - */ -function normalizeNonNegativeInt(value: number | undefined, fallback: number): number { - return typeof value === 'number' && Number.isFinite(value) && value >= 0 - ? Math.floor(value) - : fallback; -} - -function asAbortError(reason: unknown): Error { - return reason instanceof Error ? reason : new Error(String(reason ?? 'aborted')); -} - -/** - * Side-effect-free read methods. ONLY these are bounded by the per-op timeout: - * rejecting a read after the bound is a clean, determinate failure (nothing was - * written), so it safely surfaces a wedged worker on the exact paths that hang - * in production. Every other method mutates persisted state and is left - * unbounded — see DEFAULT_OPERATION_TIMEOUT_MS for why timing out a mutation - * would be unsafe with the current call sites. - */ -const READ_ONLY_METHODS = new Set([ - 'query', 'hasGraph', 'listGraphs', 'countQuads', -]); - -/** Rejection raised when a read op exceeds its per-op timeout. */ -export interface OxigraphWorkerTimeoutError extends Error { - code: 'OXIGRAPH_WORKER_OP_TIMEOUT'; - /** Which store method timed out. */ - method: string; - /** The bound that was exceeded. */ - timeoutMs: number; -} - -/** - * Explicit worker POLICY state, replacing the old cluster of interdependent - * booleans/promises that all had to agree. `respawnGaveUp` folds into the - * 'gave_up' state and the new terminal 'in_memory_lost' state; `workerExited` - * stays as a separate lower-level FACT (it is orthogonal — see the field - * comment), and `closePromise`/`respawnPromise`/`consecutiveRespawnFailures` - * still carry their own orthogonal data. A single discriminated field makes the - * invalid states that used to be representable — e.g. "gave up" AND "live", or - * an in-memory store silently marked healthy after a crash — impossible to - * construct, and it gives every read a single, obvious source of truth. The - * state graph is: - * - * live ──unexpected exit, persisted──▶ respawning ──spawned──▶ live - * live ──unexpected exit, in-memory──▶ in_memory_lost (terminal — finding 1) - * respawning ──crash-loop bound hit──▶ gave_up (terminal) - * live | respawning ──close()────────▶ closing ──drained──▶ closed (terminal) - * - * `respawning` is the only state in which the current worker thread is dead but - * a replacement is on the way, so parked ops wait it out (see callAfterRespawn). - * Every other non-`live` state means "no usable worker" and fails ops fast. - */ -type WorkerLifecycle = - | 'initializing' - | 'live' - | 'respawning' - | 'closing' - | 'closed' - | 'gave_up' - | 'in_memory_lost'; - -/** Terminal states: the store will never serve another op and never respawns. */ -const TERMINAL: ReadonlySet = new Set([ - 'closed', 'gave_up', 'in_memory_lost', -]); - -export class OxigraphWorkerStore implements TripleStore { - readonly queryCancellation = 'interruptible' as const; - - // Assigned by spawnWorker(), which the constructor always calls — hence the - // definite-assignment assertion instead of an initializer. - private worker!: Worker; - private nextId = 0; - private pending = new Map void; reject: (e: Error) => void }>(); - // #1609: per-graph write generations, bumped client-side after each - // successful mutation RPC (the worker owns no caller-visible caches). - // Feeds the chain-reconcile negative memo via `asGraphWriteGenSource`. - private readonly writeGen = new GraphWriteGenTracker(); - private readonly operationTimeoutMs: number; - /** Resolved path of the compiled worker impl; reused verbatim on respawn. */ - private readonly workerPath: string; - /** Persistence file handed to every spawned worker (undefined = in-memory). */ - private readonly persistPath: string | undefined; - /** - * Single source of truth for the store's high-level POLICY state (see - * WorkerLifecycle). It subsumes the former `respawnGaveUp` boolean (⇔ state - * === 'gave_up') and adds the terminal 'in_memory_lost' state that finding-1 - * needs, so the give-up/close/data-lost verdicts can never disagree the way - * an ad-hoc cluster of interdependent booleans could. Set to 'live' the - * moment spawnWorker() arms a fresh thread; only ever moves along the - * documented transitions (every write is guarded — `setLifecycle` for the - * non-spawn hops and `markSpawnedLive` for the spawn → 'live' hop — so a - * terminal state can never silently reopen). - * - * `workerExited` below stays as a separate, lower-level FACT ("the current - * thread has exited") because it is genuinely orthogonal to policy: during a - * graceful close() the state is 'closing' while the thread is still alive and - * mid-flush, then the thread exits — same policy state, different fact. The - * old soup was the *interdependence* of workerExited + respawnGaveUp + - * respawnPromise + closePromise all having to agree; folding the policy half - * into one discriminated field removes that. - * - * The initializer is a pre-construction placeholder only: the constructor - * always calls spawnWorker() (whose `markSpawnedLive` accepts the - * non-terminal 'initializing' placeholder), so no op ever observes this - * value. 'initializing' is deliberately NON-terminal so the spawn guard can - * enforce terminal permanence uniformly — a placeholder of 'closed' (a - * terminal state) would have made the first legal spawn indistinguishable - * from illegally reopening a closed store. - */ - private lifecycle: WorkerLifecycle = 'initializing'; - /** Set once the CURRENT worker thread has exited (graceful close, crash, or kill); cleared by spawnWorker(). */ - private workerExited = false; - /** Memoized close so repeat/concurrent close() calls share one teardown. */ - private closePromise: Promise | null = null; - /** - * In-flight replacement of a crashed worker. While set, NEW ops park on it - * (see callWithTimeout) instead of failing, so a request that races the - * respawn sees a slightly slower success rather than a spurious error. - * Null whenever a live worker is armed. Tracked separately from `lifecycle` - * because ops need the actual promise to await, not just the 'respawning' tag. - */ - private respawnPromise: Promise | null = null; - /** - * How many respawned workers in a row died without answering a single op - * successfully. Reset to 0 by the first successful reply; feeds the backoff - * tier and the MAX_CONSECUTIVE_RESPAWNS give-up bound. - */ - private consecutiveRespawnFailures = 0; - - constructor(persistPath?: string, opts?: OxigraphWorkerStoreOptions) { - this.operationTimeoutMs = normalizeNonNegativeInt(opts?.operationTimeoutMs, DEFAULT_OPERATION_TIMEOUT_MS); - - // Resolve the worker impl with a small search path so this keeps - // working in all three deployment shapes we actually run in: - // - // 1. Production / npm install / built monorepo — this module is - // loaded from `dist/adapters/oxigraph-worker.js`, so the - // sibling `./oxigraph-worker-impl.js` resolves correctly. - // 2. vitest against raw source — this module is loaded from - // `src/adapters/oxigraph-worker.ts`, so the sibling - // `./oxigraph-worker-impl.js` does NOT exist, but its compiled - // twin in `dist/adapters/` does as long as the caller ran - // `pnpm --filter ...dkg-storage build` first. Redirect to - // that path so the adapter is runnable in dev loops. - // 3. Neither file exists — genuinely unbuilt tree. Throw a loud, - // actionable error explaining the fix (`pnpm build`), matching - // the expectation in `test/storage.test.ts`. - const siblingJsUrl = new URL('./oxigraph-worker-impl.js', import.meta.url); - const siblingJsPath = fileURLToPath(siblingJsUrl); - let workerPath: string | null = existsSync(siblingJsPath) ? siblingJsPath : null; - if (!workerPath) { - const srcAdapters = `${sep}src${sep}adapters${sep}`; - const distAdapters = `${sep}dist${sep}adapters${sep}`; - if (siblingJsPath.includes(srcAdapters)) { - const candidate = siblingJsPath.replace(srcAdapters, distAdapters); - if (existsSync(candidate)) workerPath = candidate; - } - } - if (!workerPath) { - throw new Error( - `oxigraph-worker adapter: compiled worker artefact ` + - `\`oxigraph-worker-impl.js\` was not found next to ` + - `${siblingJsPath} or in the sibling \`dist/adapters/\` ` + - `directory. Run \`pnpm --filter @origintrail-official/dkg-storage build\` ` + - `before using this adapter.`, - ); - } - this.workerPath = workerPath; - this.persistPath = persistPath; - this.spawnWorker(); - } - - /** - * The single writer for `this.lifecycle`. A typed setter (rather than raw - * field assignment scattered across the class) gives us one place to assert - * the invariant that matters: a TERMINAL state is a dead end. Without this - * guard a latent bug — a stray respawn resurrecting a `gave_up` store, or a - * crash handler firing after `close()` — could silently re-open a store the - * operator was told is gone, which is exactly the class of "healthy-looking - * but wrong" state finding 1 is about. If we ever try to leave a terminal - * state we throw loudly instead. - */ - private setLifecycle(next: WorkerLifecycle): void { - if (this.lifecycle === next) return; - if (TERMINAL.has(this.lifecycle)) { - throw new Error( - `oxigraph-worker: illegal lifecycle transition ${this.lifecycle} → ${next} ` + - '(terminal states are permanent) — this is a bug in the respawn supervisor.', - ); - } - this.lifecycle = next; - } - - /** - * The ONE guarded transition INTO 'live' — used by spawnWorker() for both the - * constructor's first spawn (from the 'initializing' placeholder) and a - * respawn (from 'respawning'). `setLifecycle` can't own this hop because - * entering 'live' is legal only from those two non-terminal states, but the - * terminal-permanence invariant must still hold: a spawn attempted after - * 'closed' / 'gave_up' / 'in_memory_lost' (e.g. a future respawn/close code - * path calling spawnWorker() by mistake) MUST throw rather than silently - * resurrect a store that promised it would never serve another op. - */ - private markSpawnedLive(): void { - if (this.lifecycle === 'live') return; - if (this.lifecycle !== 'initializing' && this.lifecycle !== 'respawning') { - throw new Error( - `oxigraph-worker: illegal spawn transition ${this.lifecycle} → live ` + - '(a worker may only be spawned from initializing or respawning; ' + - 'terminal states are permanent) — this is a bug in the respawn supervisor.', - ); - } - this.lifecycle = 'live'; - } - - /** True once an operator-initiated close() has begun (closing or closed). */ - private get isClosing(): boolean { - return this.lifecycle === 'closing' || this.lifecycle === 'closed'; - } - - /** - * Create the worker thread and arm its lifecycle handlers. Shared by the - * constructor and by respawn() so a replacement worker is wired IDENTICALLY - * to the original — same impl path, same workerData, same handlers — and the - * OxigraphWorkerStore object keeps its identity, so every alias held across - * the daemon (routes, publisher, gossip ingest) transparently talks to the - * new thread. Each handler captures the worker it was armed for and no-ops - * if `this.worker` has since moved on, so a stale event from an - * already-replaced thread can never clobber the live one's state. - */ - private spawnWorker(): void { - const worker = new Worker(this.workerPath, { - workerData: { persistPath: this.persistPath }, - }); - this.worker = worker; - this.workerExited = false; - // Arming a fresh thread IS the transition into 'live'. Route it through the - // guarded `markSpawnedLive` (not `setLifecycle`, which forbids ALL entries - // to 'live'): it permits the two legal predecessors — the 'initializing' - // placeholder (constructor's first spawn) and 'respawning' (a respawn) — - // while still throwing on terminal states, so a stray spawn after close / - // give-up / in-memory-loss can never silently resurrect the store. close() - // and the respawn supervisor are the only other writers and never race - // this: a spawn only happens in the constructor or within respawn(), which - // bails the moment a close() is seen. - this.markSpawnedLive(); - worker.on('message', (msg: { id: number; result?: unknown; error?: string }) => { - if (this.worker !== worker) return; - // Any successful reply proves this worker is healthy, which ends the - // crash-loop accounting window (see MAX_CONSECUTIVE_RESPAWNS). - if (!msg.error) this.consecutiveRespawnFailures = 0; - const p = this.pending.get(msg.id); - if (!p) return; - this.pending.delete(msg.id); - if (msg.error) p.reject(new Error(msg.error)); - else p.resolve(msg.result); - }); - worker.on('error', (err) => { - if (this.worker !== worker) return; - // Loud by design. 'error' means the worker thread threw an uncaught - // exception (an 'exit' event follows). This used to be silent, which is - // how a testnet node served HTTP for DAYS with a dead store — green - // /api/status, 503 on every write — and nothing in the logs said why. - console.error('[oxigraph-worker] worker thread error (thread is going down):', err); - for (const [, p] of this.pending) p.reject(err); - this.pending.clear(); - }); - // Once the worker exits, no pending op can ever get a reply. Settle them all - // (reject) instead of leaving callers hung forever — this is what makes the - // unbounded `close()` safe: if a second close (or any op) is still queued - // when terminate() kills the thread, it rejects here rather than hanging. - // - // Three very different exits land here (branch on lifecycle + persistPath): - // • intentional — close() ran (state is 'closing'). Stay closed; close() - // owns the final transition to 'closed'. - // • unexpected, disk-persisted — the thread died on its own. - // ERR_WORKER_OUT_OF_MEMORY, for example, kills ONLY the worker thread, - // not the process, so the daemon would otherwise keep serving with a - // permanently dead store. The committed state is on disk, so a fresh - // worker on the same path reopens it — go 'respawning' and recover. - // • unexpected, IN-MEMORY — there is no disk to reload from, so a fresh - // worker would come up EMPTY and look perfectly healthy while every - // row written before the crash is silently gone (confirmed live: data - // vanished). That is unrecoverable, so we FAIL CLOSED into the terminal - // 'in_memory_lost' state instead of respawning an empty store — every - // later op then rejects with a clear data-loss error (see postToWorker) - // rather than returning wrong-but-plausible results. - worker.on('exit', (code) => { - if (this.worker !== worker) return; - this.workerExited = true; - const intentional = this.isClosing; - // Distinguish the two unexpected exits up front so the log and the - // lifecycle transition below agree on what just happened. - const inMemoryLost = !intentional && this.persistPath === undefined; - if (intentional) { - console.info(`[oxigraph-worker] worker exited (code ${code}) — initiated by close()`); - } else if (inMemoryLost) { - console.error( - `[oxigraph-worker] worker exited UNEXPECTEDLY (code ${code}) — nobody called close(). ` + - 'FATAL: this store is IN-MEMORY (no persistPath), so its entire contents were lost with the ' + - 'worker thread and CANNOT be recovered. Failing the store closed instead of silently continuing ' + - 'with an empty store — every store-backed request will now fail fast. Use a disk-persisted store ' + - '(store.path) if the workload must survive a worker crash.', - ); - } else { - console.error( - `[oxigraph-worker] worker exited UNEXPECTEDLY (code ${code}) — nobody called close(). ` + - 'The store is disk-persisted; respawning a fresh worker on the same path.', - ); - } - if (this.pending.size > 0) { - const err = intentional - ? new Error('oxigraph-worker: worker exited before the operation completed (store closed)') - : inMemoryLost - ? new Error( - `oxigraph-worker: the IN-MEMORY worker exited unexpectedly (code ${code}) before the operation ` + - 'completed and its data was lost — an in-memory store cannot be recovered from a worker crash.', - ) - : new Error( - `oxigraph-worker restarted — retry: the worker exited unexpectedly (code ${code}) before the ` + - 'operation completed, so its outcome is unknown; a replacement worker is being spawned.', - ); - for (const [, p] of this.pending) p.reject(err); - this.pending.clear(); - } - // Route the state transition. close() already moved us to 'closing' and - // owns the final hop to 'closed', so the intentional case touches nothing. - if (inMemoryLost) { - this.setLifecycle('in_memory_lost'); - } else if (!intentional) { - this.scheduleRespawn(); - } - }); - } - - /** Arm (at most one) background respawn; the policy lives in respawn(). */ - private scheduleRespawn(): void { - // Never resurrect a store that has already given up, been closed, or lost - // its in-memory data, and never stack a second supervisor on an in-flight - // one. This is only ever reached from the exit handler's disk-persisted - // unexpected-exit branch, so we're normally transitioning out of 'live'. - if (this.respawnPromise || TERMINAL.has(this.lifecycle)) return; - this.setLifecycle('respawning'); - // respawn() never rejects (every failure path is caught and looped or - // latched), so this promise can't become an unhandled rejection while no - // op happens to be parked on it. - this.respawnPromise = this.respawn().finally(() => { - this.respawnPromise = null; - }); - } - - /** - * Replace a crashed worker with a fresh one: immediate first attempt, then - * capped backoff (RESPAWN_BACKOFF_MS) between consecutive attempts, giving - * up for good after MAX_CONSECUTIVE_RESPAWNS workers in a row die without - * serving a single successful op. Giving up latches the store closed — - * exactly the pre-recovery behaviour — but with a FATAL log telling the - * operator what happened and where to look, instead of the old silence. - */ - private async respawn(): Promise { - while (true) { - // close() raced the respawn — honour it. An operator-initiated close - // must never be resurrected by the supervisor. (close() flips lifecycle - // to 'closing' atomically before awaiting, so isClosing is the signal.) - if (this.isClosing) return; - if (this.consecutiveRespawnFailures >= MAX_CONSECUTIVE_RESPAWNS) { - this.setLifecycle('gave_up'); - console.error( - `[oxigraph-worker] FATAL: the worker died ${MAX_CONSECUTIVE_RESPAWNS} times in a row without ` + - 'serving a single successful operation — giving up on respawn and latching the store closed. ' + - 'Every store-backed request will now fail fast. Investigate the crash cause (worker OOM → raise ' + - 'memory or move to an external SPARQL backend via store.backend "sparql-http" / "blazegraph"; ' + - 'corrupt persist file → see the quarantine log) and restart the node.', - ); - return; - } - const attempt = this.consecutiveRespawnFailures; - this.consecutiveRespawnFailures += 1; - const delayMs = RESPAWN_BACKOFF_MS[Math.min(attempt, RESPAWN_BACKOFF_MS.length - 1)]; - if (delayMs > 0) { - console.error( - `[oxigraph-worker] respawn attempt ${attempt + 1}/${MAX_CONSECUTIVE_RESPAWNS} in ${delayMs}ms…`, - ); - await sleep(delayMs); - // Re-check: close() may have arrived during the backoff sleep. - if (this.isClosing) return; - } - try { - this.spawnWorker(); - console.info(`[oxigraph-worker] respawned worker (attempt ${attempt + 1}/${MAX_CONSECUTIVE_RESPAWNS})`); - return; - } catch (err) { - // new Worker() itself can throw (e.g. the impl file vanished at - // runtime). Treat it exactly like a worker that died instantly and - // loop into the next backoff tier. - console.error('[oxigraph-worker] respawn attempt failed to start a worker:', err); - } - } - } - - private call(method: string, ...args: unknown[]): Promise { - // Only read-only ops are bounded; mutations run unbounded (timeoutMs 0) so a - // timed-out write is never reported as a clean failure while still in flight. - const timeoutMs = READ_ONLY_METHODS.has(method) ? this.operationTimeoutMs : 0; - return this.callWithTimeout(timeoutMs, undefined, method, ...args); - } - - /** - * Post one op to the (live) worker and await its reply, bounding the caller's - * wait by `timeoutMs` (0 = wait indefinitely). The bound is per-CALLER: on - * timeout or caller abort we reject and drop the pending entry, but the - * single-threaded worker is STILL running the op — the late reply is then - * ignored (the message handler no-ops on a missing id) rather than - * double-settling this promise. Only read-only ops are ever given a non-zero - * timeout (see `call`), so a fired timeout is always a determinate, - * side-effect-free failure. If a crashed worker is being replaced, the op - * first parks on the respawn (the timeout starts only once it is actually - * posted — backoff is capped well below the default read bound anyway). - */ - private callWithTimeout( - timeoutMs: number, - signal: AbortSignal | undefined, - method: string, - ...args: unknown[] - ): Promise { - // A crashed worker may be mid-replacement right now. Park the op on the - // respawn instead of failing it: the store is disk-persisted and about to - // come back, so the caller sees a slightly slower success rather than a - // spurious "store is closed" for a condition the store is already fixing. - if (this.respawnPromise) return this.callAfterRespawn(timeoutMs, signal, method, args); - return this.postToWorker(timeoutMs, signal, method, args); - } - - /** - * Wait out the in-flight respawn(s), then post as normal. A loop rather - * than a single await because the replacement worker can itself die before - * this op gets posted, arming a new respawnPromise. If the respawn gave up, - * postToWorker's workerExited guard turns this into the fail-fast closed - * error (with the crash-loop guidance appended). - */ - private async callAfterRespawn( - timeoutMs: number, - signal: AbortSignal | undefined, - method: string, - args: unknown[], - ): Promise { - while (this.respawnPromise) await this.respawnPromise; - return this.postToWorker(timeoutMs, signal, method, args); - } - - private postToWorker( - timeoutMs: number, - signal: AbortSignal | undefined, - method: string, - args: unknown[], - ): Promise { - const id = this.nextId++; - return new Promise((resolve, reject) => { - // The worker is gone (closed, crashed with respawn given up, or an - // in-memory store whose data was lost) — a posted message would never be - // answered, so fail fast instead of registering a pending entry that can - // only ever hang. The lifecycle picks the RIGHT diagnostic: - // • in_memory_lost — the data is unrecoverable; say so plainly instead - // of pretending the store merely "closed" (finding 1: never let an - // in-memory crash look like a clean close or an empty-but-healthy store). - // • gave_up — closed + the crash-loop guidance. - // • otherwise — a plain operator close. - if (this.workerExited) { - if (this.lifecycle === 'in_memory_lost') { - reject(new Error( - `oxigraph-worker: cannot run "${method}" — the IN-MEMORY store's worker crashed and its data was ` + - 'lost. An in-memory store cannot be recovered from a worker crash; every request now fails ' + - 'fast. Use a disk-persisted store (store.path) or restart the node to start from empty.', - )); - return; - } - reject(new Error( - `oxigraph-worker: cannot run "${method}" — the store is closed.` + - (this.lifecycle === 'gave_up' - ? ' (The worker crashed repeatedly and automatic respawn gave up — restart the node and ' + - 'investigate the [oxigraph-worker] crash logs.)' - : ''), - )); - return; - } - if (signal?.aborted) { - reject(asAbortError(signal.reason)); - return; - } - let timer: ReturnType | undefined; - let onAbort: (() => void) | undefined; - const cleanup = () => { - if (timer) clearTimeout(timer); - if (signal && onAbort) signal.removeEventListener('abort', onAbort); - }; - if (timeoutMs > 0) { - timer = setTimeout(() => { - if (this.pending.delete(id)) { - cleanup(); - const err = new Error( - `oxigraph-worker: "${method}" timed out after ${timeoutMs}ms. ` + - `The embedded store runs on a single worker thread, so a long-running or ` + - `stuck operation blocks all reads queued behind it. For heavy workloads ` + - `point the node at an external SPARQL server (store.backend "sparql-http" ` + - `/ "blazegraph"), or raise / disable store.options.operationTimeoutMs.`, - ) as OxigraphWorkerTimeoutError; - err.code = 'OXIGRAPH_WORKER_OP_TIMEOUT'; - err.method = method; - err.timeoutMs = timeoutMs; - reject(err); - } - }, timeoutMs); - // A pending-op timer must not keep the process alive on its own. - if (typeof timer.unref === 'function') timer.unref(); - } - this.pending.set(id, { - resolve: (v) => { cleanup(); resolve(v as T); }, - reject: (e) => { cleanup(); reject(e); }, - }); - if (signal) { - onAbort = () => { - if (this.pending.delete(id)) { - cleanup(); - reject(asAbortError(signal.reason)); - } - }; - signal.addEventListener('abort', onAbort, { once: true }); - if (signal.aborted) { - onAbort(); - return; - } - } - try { - this.worker.postMessage({ id, method, args }); - } catch (err) { - if (this.pending.delete(id)) { - cleanup(); - reject(err instanceof Error ? err : new Error(String(err))); - } - } - }); - } - - // A single atomic worker message: all quads commit together or the call - // fails. The contract is "all-or-nothing" and every caller can rely on it - // (e.g. FinalizationHandler packs both canonical copies into one insert so one - // can't land without the other). Large idempotent bulk imports that want - // head-of-line fairness should run on an external SPARQL server, not by - // silently fragmenting this insert into non-atomic chunks. - async insert(quads: Quad[]): Promise { - await this.call('insert', quads); - this.writeGen.recordGraphWrites(new Set(quads.map((q) => q.graph || ''))); - } - async delete(quads: Quad[]): Promise { - await this.call('delete', quads); - this.writeGen.recordGraphWrites(new Set(quads.map((q) => q.graph || ''))); - } - async deleteByPattern(pattern: Partial): Promise { - const removed = await this.call('deleteByPattern', pattern); - if (pattern.graph) this.writeGen.recordGraphWrites([pattern.graph]); - else this.writeGen.recordUnscopedWrite(); - return removed; - } - // Server-side SPARQL UPDATE forwarded to the worker's OxigraphStore (which - // implements `update`); same atomic single-message contract as `insert`. - // eslint-disable-next-line @typescript-eslint/no-unused-vars - async update(sparql: string, _options?: UpdateOptions): Promise { - await this.call('update', sparql); - // A raw UPDATE's write scope is not derivable at the call site - // (`touchedGraphs` hints only membership changes) — unscoped bump. - this.writeGen.recordUnscopedWrite(); - } - async query(sparql: string, options?: TripleStoreQueryOptions): Promise { - return this.callWithTimeout(this.operationTimeoutMs, options?.signal, 'query', sparql); - } - async hasGraph(graphUri: string, options?: TripleStoreQueryOptions): Promise { - return this.callWithTimeout(this.operationTimeoutMs, options?.signal, 'hasGraph', graphUri); - } - async createGraph(graphUri: string): Promise { return this.call('createGraph', graphUri); } - async dropGraph(graphUri: string): Promise { - await this.call('dropGraph', graphUri); - this.writeGen.recordGraphWrites([graphUri]); - } - async listGraphs(options?: TripleStoreQueryOptions): Promise { - return this.callWithTimeout(this.operationTimeoutMs, options?.signal, 'listGraphs'); - } - async deleteBySubjectPrefix(graphUri: string, prefix: string): Promise { - const removed = await this.call('deleteBySubjectPrefix', graphUri, prefix); - this.writeGen.recordGraphWrites([graphUri]); - return removed; - } - - /** {@link GraphWriteGenSource} capability (#1609) — see graph-write-gen.ts. */ - getWriteGen(graphPrefix: string): number { - return this.writeGen.getWriteGen(graphPrefix); - } - async countQuads(graphUri?: string): Promise { return this.call('countQuads', graphUri); } - async flush(): Promise { return this.call('flush'); } - - async close(): Promise { - // Memoized + serialized: every close() call shares ONE teardown promise, so - // we issue exactly one close RPC. (A second unbounded close RPC would be - // orphaned when terminate() kills the worker after the first resolves, and - // — without the exit handler — would hang forever.) - // - // Flip the lifecycle to 'closing' FIRST (before awaiting anything), unless - // the store already reached a terminal state on its own (gave_up / - // in_memory_lost, or an even earlier close). Doing it synchronously here is - // what lets an in-flight respawn see the close and bail (respawn() checks - // isClosing) and what makes the worker's 'exit' handler treat this exit as - // intentional. A store that already latched terminal is simply reaped below - // — its state is permanent, so we must NOT try to move it to 'closing'. - if (!this.closePromise) { - if (!TERMINAL.has(this.lifecycle)) this.setLifecycle('closing'); - this.closePromise = this.doClose(); - } - return this.closePromise; - } - - private async doClose(): Promise { - // Worker already gone (crash/kill, respawn give-up, or in-memory loss) — - // there's nothing to flush; just make sure the thread is reaped. Keeps - // close() idempotent and non-throwing. Only settle into the terminal - // 'closed' state when we're not already in a *different* terminal state - // (gave_up / in_memory_lost stay as-is so their diagnostics survive). - if (this.workerExited) { - try { await this.worker.terminate(); } catch { /* already terminated */ } - if (!TERMINAL.has(this.lifecycle)) this.setLifecycle('closed'); - return; - } - // `close` runs the worker's FINAL synchronous flush (insert() only schedules - // a 50ms debounced flush, so close is what guarantees durability). It is - // therefore EXEMPT from the per-op timeout (timeoutMs 0): bounding it could - // fire the timeout while the worker is mid-flush, and the `finally` would - // then terminate() the thread before pending writes hit disk — losing data. - // terminate() always runs in `finally`; the worker's 'exit' handler then - // rejects anything still pending, so nothing leaks or hangs. - try { - await this.callWithTimeout(0, undefined, 'close'); - } finally { - await this.worker.terminate(); - // The 'exit' handler above left us in 'closing' (intentional exit). Land - // the terminal transition here so a post-close op fails fast (workerExited - // is now set, and the guard reads 'closed' for the plain message). - if (!TERMINAL.has(this.lifecycle)) this.setLifecycle('closed'); - } - } -} - -registerTripleStoreAdapter('oxigraph-worker', async (opts) => { - const filePath = opts?.path as string | undefined; - return new OxigraphWorkerStore(filePath, { - operationTimeoutMs: - typeof opts?.operationTimeoutMs === 'number' ? (opts.operationTimeoutMs as number) : undefined, - }); -}); diff --git a/packages/storage/src/adapters/oxigraph.ts b/packages/storage/src/adapters/oxigraph.ts index f476b3176d..d5c9aa8522 100644 --- a/packages/storage/src/adapters/oxigraph.ts +++ b/packages/storage/src/adapters/oxigraph.ts @@ -254,8 +254,8 @@ export class OxigraphStore implements TripleStore { async query(sparql: string, options?: TripleStoreQueryOptions): Promise { throwIfAborted(options?.signal); // The embedded Oxigraph binding executes synchronously, so a caller abort - // cannot interrupt this native call mid-flight. Use oxigraph-worker or an - // HTTP backend when long sync queries need prompt cancellation. + // cannot interrupt this native call mid-flight. Use an HTTP backend when + // long sync queries need prompt cancellation. const result = this.store.query(sparql); throwIfAborted(options?.signal); diff --git a/packages/storage/src/index.ts b/packages/storage/src/index.ts index 5e8f4ee97c..c916a1843f 100644 --- a/packages/storage/src/index.ts +++ b/packages/storage/src/index.ts @@ -10,16 +10,35 @@ export { type AskResult, type TripleStoreConfig, type TripleStoreBackend, + type TripleStoreFactoryConfig, type TripleStoreQueryOptions, type UpdateOptions, type LargeLiteralStorageConfig, registerTripleStoreAdapter, createTripleStore, + toTripleStoreBackend, tryUpdateWithTouchedGraphs, - isExternalBackend, getSparqlEndpoint, type SparqlEndpoint, + type SparqlEndpointStoreConfig, } from './triple-store.js'; +export { + STORAGE_ADAPTERS, + classifyTripleStoreBackend, + customTripleStoreBackend, + getStorageAdapterPolicy, + isExternalBackend, + isStorageAdapterBackend, + storageAdapterNames, + type ClassifiedTripleStoreBackend, + type CustomTripleStoreBackend, + type ExternalStoreBackend, + type LocalStoreBackend, + type StorageAdapterBackend, + type StorageAdapterKind, + type StorageAdapterOfKind, + type StorageAdapterPolicy, +} from './store-backends.js'; export { StorePriorityScheduler, externalStorePriorityScheduler, @@ -60,7 +79,6 @@ export { } from './graph-write-gen.js'; export { OxigraphStore } from './adapters/oxigraph.js'; -export { OxigraphWorkerStore } from './adapters/oxigraph-worker.js'; export { BlazegraphStore } from './adapters/blazegraph.js'; export { SparqlHttpStore, @@ -87,6 +105,5 @@ export { PrivateContentStore } from './private-store.js'; // Side-effect: register built-in adapters import './adapters/oxigraph.js'; -import './adapters/oxigraph-worker.js'; import './adapters/blazegraph.js'; import './adapters/sparql-http.js'; diff --git a/packages/storage/src/store-backends.ts b/packages/storage/src/store-backends.ts new file mode 100644 index 0000000000..0d07ba1b2b --- /dev/null +++ b/packages/storage/src/store-backends.ts @@ -0,0 +1,84 @@ +/** + * Canonical metadata for adapters the storage factory can construct. + * + * Daemon defaults, retired config names, migration policy, and CLI labels live + * in the CLI package. Keeping this registry adapter-only prevents the storage + * layer from acquiring daemon lifecycle or presentation policy. + */ +export const STORAGE_ADAPTERS = { + oxigraph: { + kind: 'local', + requiresExistingPath: false, + }, + 'oxigraph-persistent': { + kind: 'local', + requiresExistingPath: true, + }, + blazegraph: { + kind: 'external', + queryEndpointOption: 'url', + updateEndpointOption: 'url', + }, + 'sparql-http': { + kind: 'external', + queryEndpointOption: 'queryEndpoint', + updateEndpointOption: 'updateEndpoint', + authOption: 'auth', + }, +} as const; + +export type StorageAdapterBackend = keyof typeof STORAGE_ADAPTERS; +export type StorageAdapterPolicy = (typeof STORAGE_ADAPTERS)[StorageAdapterBackend]; +export type StorageAdapterKind = StorageAdapterPolicy['kind']; +export type StorageAdapterOfKind = { + [Backend in StorageAdapterBackend]: typeof STORAGE_ADAPTERS[Backend] extends { kind: Kind } + ? Backend + : never; +}[StorageAdapterBackend]; +export type ExternalStoreBackend = StorageAdapterOfKind<'external'>; +export type LocalStoreBackend = StorageAdapterOfKind<'local'>; + +declare const CUSTOM_TRIPLE_STORE_BACKEND: unique symbol; +export type CustomTripleStoreBackend = string & { + readonly [CUSTOM_TRIPLE_STORE_BACKEND]: true; +}; + +export type ClassifiedTripleStoreBackend = + | { kind: 'adapter'; backend: StorageAdapterBackend } + | { kind: 'custom'; backend: CustomTripleStoreBackend }; + +export function storageAdapterNames(): StorageAdapterBackend[] { + return Object.keys(STORAGE_ADAPTERS) as StorageAdapterBackend[]; +} + +export function isStorageAdapterBackend( + backend: string | undefined | null, +): backend is StorageAdapterBackend { + return backend != null && Object.prototype.hasOwnProperty.call(STORAGE_ADAPTERS, backend); +} + +export function getStorageAdapterPolicy( + backend: string | undefined | null, +): StorageAdapterPolicy | undefined { + return isStorageAdapterBackend(backend) ? STORAGE_ADAPTERS[backend] : undefined; +} + +export function isExternalBackend( + backend: string | undefined | null, +): backend is ExternalStoreBackend { + return isStorageAdapterBackend(backend) && STORAGE_ADAPTERS[backend].kind === 'external'; +} + +export function customTripleStoreBackend(backend: string): CustomTripleStoreBackend { + if (!backend.trim()) throw new Error('Custom triple-store backend name cannot be empty'); + if (isStorageAdapterBackend(backend)) { + throw new Error(`Known triple-store adapter "${backend}" does not need a custom-backend wrapper`); + } + return backend as CustomTripleStoreBackend; +} + +export function classifyTripleStoreBackend(backend: string): ClassifiedTripleStoreBackend { + return isStorageAdapterBackend(backend) + ? { kind: 'adapter', backend } + : { kind: 'custom', backend: backend as CustomTripleStoreBackend }; +} diff --git a/packages/storage/src/triple-store.ts b/packages/storage/src/triple-store.ts index 329c91e529..9ad2a779b7 100644 --- a/packages/storage/src/triple-store.ts +++ b/packages/storage/src/triple-store.ts @@ -17,6 +17,15 @@ import { ChangelogStore, type ChangelogStoreOptions, } from './changelog-store.js'; +import { + classifyTripleStoreBackend, + getStorageAdapterPolicy, + isExternalBackend, + type CustomTripleStoreBackend, + type StorageAdapterBackend, +} from './store-backends.js'; + +export { isExternalBackend } from './store-backends.js'; export interface Quad { subject: string; @@ -166,11 +175,19 @@ export async function tryUpdateWithTouchedGraphs( return true; } -export type TripleStoreBackend = 'oxigraph' | 'oxigraph-persistent' | 'oxigraph-worker' | 'blazegraph' | 'sparql-http' | string; +/** + * Backends the storage factory can construct: registered built-in adapters or + * an explicitly branded custom adapter name. + */ +export type TripleStoreBackend = StorageAdapterBackend | CustomTripleStoreBackend; + +/** Explicitly cross from stringly daemon/user config into the factory model. */ +export function toTripleStoreBackend(backend: string): TripleStoreBackend { + const classification = classifyTripleStoreBackend(backend); + return classification.backend; +} -// Backends that talk to a remote SPARQL endpoint over HTTP rather than -// owning local files. The local/external split governs three pieces of -// daemon behaviour: +// The canonical backend taxonomy governs three pieces of daemon behaviour: // 1. Store-size metric — local backends report file bytes, external // backends have no file to stat (`getStoreBytes` returns null). // 2. Chain-reset wipe — local backends `rm` files, external backends @@ -181,12 +198,6 @@ export type TripleStoreBackend = 'oxigraph' | 'oxigraph-persistent' | 'oxigraph- // 3. Boot health check — for external backends, an ASK probe runs at // daemon start; an unreachable endpoint exits the daemon with an // actionable message rather than booting half-broken. -const EXTERNAL_BACKENDS: ReadonlySet = new Set(['blazegraph', 'sparql-http']); - -export function isExternalBackend(backend: string | undefined | null): boolean { - return typeof backend === 'string' && EXTERNAL_BACKENDS.has(backend); -} - /** * Shape-normalised SPARQL endpoint extracted from a TripleStoreConfig. * @@ -202,34 +213,39 @@ export interface SparqlEndpoint { headers: Record; } -export function getSparqlEndpoint(storeConfig: TripleStoreConfig): SparqlEndpoint { +export interface SparqlEndpointStoreConfig { + backend: string; + options?: Record; +} + +export function getSparqlEndpoint(storeConfig: SparqlEndpointStoreConfig): SparqlEndpoint { if (!isExternalBackend(storeConfig.backend)) { throw new Error( `getSparqlEndpoint called for non-external backend "${storeConfig.backend}"`, ); } const opts = (storeConfig.options ?? {}) as Record; - if (storeConfig.backend === 'blazegraph') { - const url = typeof opts.url === 'string' ? opts.url : ''; - if (!url) { - throw new Error('blazegraph storeConfig requires options.url'); - } - return { queryUrl: url, updateUrl: url, headers: {} }; + const policy = getStorageAdapterPolicy(storeConfig.backend); + if (!policy || policy.kind !== 'external') { + throw new Error(`No external-store policy found for "${storeConfig.backend}"`); } - // sparql-http - const queryEndpoint = typeof opts.queryEndpoint === 'string' ? opts.queryEndpoint : ''; - if (!queryEndpoint) { - throw new Error('sparql-http storeConfig requires options.queryEndpoint'); + const queryOption = policy.queryEndpointOption; + const queryUrl = typeof opts[queryOption] === 'string' ? opts[queryOption] as string : ''; + if (!queryUrl) { + throw new Error(`${storeConfig.backend} storeConfig requires options.${queryOption}`); } - const updateEndpoint = - typeof opts.updateEndpoint === 'string' && opts.updateEndpoint - ? opts.updateEndpoint - : queryEndpoint; + const updateOption = policy.updateEndpointOption; + const updateUrl = typeof opts[updateOption] === 'string' && opts[updateOption] + ? opts[updateOption] as string + : queryUrl; const headers: Record = {}; - if (typeof opts.auth === 'string' && opts.auth) { - headers['Authorization'] = opts.auth; + if ('authOption' in policy) { + const auth = opts[policy.authOption]; + if (typeof auth === 'string' && auth) { + headers.Authorization = auth; + } } - return { queryUrl: queryEndpoint, updateUrl: updateEndpoint, headers }; + return { queryUrl, updateUrl, headers }; } export interface LargeLiteralStorageConfig { @@ -256,22 +272,40 @@ export interface TripleStoreConfig { changelog?: boolean | ChangelogStoreOptions; } +export type TripleStoreFactoryConfig = TripleStoreConfig; + type AdapterFactory = ( options?: Record, ) => Promise; const adapterRegistry = new Map(); -export function registerTripleStoreAdapter( - name: string, +// Runtime compatibility guard for callers compiled before the worker adapter +// was removed. This is intentionally not part of STORAGE_ADAPTERS: it provides +// a migration error without making retired daemon/config policy constructible. +const REMOVED_ADAPTER_GUIDANCE: Readonly> = { + 'oxigraph-worker': + 'Use "sparql-http" or "blazegraph" for an HTTP store, or ' + + '"oxigraph-persistent" for embedded persistence.', +}; + +export function registerTripleStoreAdapter( + name: Backend, factory: AdapterFactory, -): void { +): Backend extends StorageAdapterBackend ? Backend : CustomTripleStoreBackend { adapterRegistry.set(name, factory); + return name as Backend extends StorageAdapterBackend ? Backend : CustomTripleStoreBackend; } export async function createTripleStore( - config: TripleStoreConfig, + config: TripleStoreFactoryConfig, ): Promise { + const removedGuidance = REMOVED_ADAPTER_GUIDANCE[config.backend]; + if (removedGuidance) { + throw new Error( + `TripleStore backend "${config.backend}" is no longer supported. ${removedGuidance}`, + ); + } const factory = adapterRegistry.get(config.backend); if (!factory) { throw new Error( @@ -337,8 +371,7 @@ function shouldEnableGraphSetIndex(config: TripleStoreConfig): boolean { function isDefaultLocalGraphSetIndexBackend(backend: TripleStoreBackend): boolean { return backend === 'oxigraph' - || backend === 'oxigraph-persistent' - || backend === 'oxigraph-worker'; + || backend === 'oxigraph-persistent'; } function resolveLargeLiteralStorageOptions( diff --git a/packages/storage/test/graph-set-index-store.test.ts b/packages/storage/test/graph-set-index-store.test.ts index 751b937fe2..67aaecb55e 100644 --- a/packages/storage/test/graph-set-index-store.test.ts +++ b/packages/storage/test/graph-set-index-store.test.ts @@ -727,8 +727,10 @@ describe('GraphSetIndexStore', () => { }); it('leaves custom backends uncached unless explicitly enabled', async () => { - const backend = 'custom-remote-graph-set-index-test'; - registerTripleStoreAdapter(backend, async () => new OxigraphStore()); + const backend = registerTripleStoreAdapter( + 'custom-remote-graph-set-index-test', + async () => new OxigraphStore(), + ); const defaultStore = await createTripleStore({ backend }); expect(defaultStore.listGraphsByPrefix).toBeUndefined(); diff --git a/packages/storage/test/is-external-backend.test.ts b/packages/storage/test/is-external-backend.test.ts index 759e08ec35..daae5378d4 100644 --- a/packages/storage/test/is-external-backend.test.ts +++ b/packages/storage/test/is-external-backend.test.ts @@ -18,7 +18,6 @@ describe('isExternalBackend', () => { it('returns false for the oxigraph family', () => { expect(isExternalBackend('oxigraph')).toBe(false); - expect(isExternalBackend('oxigraph-worker')).toBe(false); expect(isExternalBackend('oxigraph-persistent')).toBe(false); }); diff --git a/packages/storage/test/oxigraph-worker-resilience.test.ts b/packages/storage/test/oxigraph-worker-resilience.test.ts deleted file mode 100644 index 89c7c6cb98..0000000000 --- a/packages/storage/test/oxigraph-worker-resilience.test.ts +++ /dev/null @@ -1,261 +0,0 @@ -import { describe, it, expect } from 'vitest'; -import { mkdtempSync, rmSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { OxigraphWorkerStore, createTripleStore, type Quad } from '../src/index.js'; - -// These exercise the embedded worker adapter's resilience guards added to stop -// a single slow/wedged store op from hanging every other store-backed request -// behind it (issues #997 / #999 / #1002 / #1005 / #1008). They need the -// compiled worker artifact (`dist/adapters/oxigraph-worker-impl.js`); if it's -// missing we fail loudly with the remediation hint rather than silently skip, -// matching `storage.test.ts`'s convention. -// -// Design note: the per-op timeout is applied to READ-ONLY ops only. A read is -// side-effect-free, so rejecting it after the bound is a clean, determinate -// failure that surfaces a wedged worker on the exact paths that hang in prod. -// Mutations are left unbounded — timing one out would only drop the caller's -// promise while the write is still in flight, which the rest of the codebase -// would mis-read as a clean failure. So the timeout tests provoke a timeout by -// queuing a READ behind a busy worker, not by bounding a write. -function makeStore(opts?: { operationTimeoutMs?: number }, persistPath?: string): OxigraphWorkerStore { - try { - return new OxigraphWorkerStore(persistPath, opts); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - if (/oxigraph-worker-impl/.test(msg)) { - throw new Error( - `oxigraph-worker adapter is not runnable — run ` + - `\`pnpm --filter @origintrail-official/dkg-storage build\` first. Underlying: ${msg}`, - ); - } - throw err; - } -} - -// For the timeout tests the worker is left mid-operation; close() forcibly -// terminates the thread, but the graceful close reply may itself time out, so -// swallow any error during teardown. -async function closeQuietly(store: OxigraphWorkerStore): Promise { - await store.close().catch(() => {}); -} - -function quads(n: number): Quad[] { - const out: Quad[] = new Array(n); - for (let i = 0; i < n; i += 1) { - out[i] = { - subject: `urn:test:s:${i}`, - predicate: 'http://schema.org/name', - object: `"v${i}"`, - graph: 'urn:test:g', - }; - } - return out; -} - -function abortDuringListenerRegistration(message: string): AbortSignal { - let aborted = false; - let reason: Error | undefined; - return { - get aborted() { - return aborted; - }, - get reason() { - return reason; - }, - addEventListener(type: string, listener: EventListenerOrEventListenerObject) { - if (type !== 'abort') return; - aborted = true; - reason = new Error(message); - if (typeof listener === 'function') listener(new Event('abort')); - else listener.handleEvent(new Event('abort')); - }, - removeEventListener: () => undefined, - dispatchEvent: () => true, - onabort: null, - } as unknown as AbortSignal; -} - -// Occupy the single worker thread with a large UNBOUNDED insert (mutations are -// not bounded by the timeout), so a read posted right after it queues behind a -// busy worker — the production "wedged worker" signature in miniature. -const BUSY_QUERY = 'ASK { GRAPH { ?s ?p ?o } }'; - -describe('OxigraphWorkerStore resilience', () => { - it('rejects a READ queued behind a busy worker after operationTimeoutMs', async () => { - // 50k inserts take hundreds of ms; a 5ms bound on the queued read must - // reject well before the worker frees up. - const store = makeStore({ operationTimeoutMs: 5 }); - try { - const busy = store.insert(quads(50_000)); // occupies the worker (unbounded) - await expect(store.query(BUSY_QUERY)).rejects.toThrow(/timed out after 5ms/); - await busy.catch(() => {}); - } finally { - await closeQuietly(store); - } - }); - - it('surfaces the timeout as a typed OXIGRAPH_WORKER_OP_TIMEOUT error', async () => { - const store = makeStore({ operationTimeoutMs: 5 }); - try { - const busy = store.insert(quads(50_000)); - const err: any = await store.query(BUSY_QUERY).then(() => null, (e) => e); - expect(err).toBeTruthy(); - expect(err.code).toBe('OXIGRAPH_WORKER_OP_TIMEOUT'); - expect(err.method).toBe('query'); - expect(err.timeoutMs).toBe(5); - await busy.catch(() => {}); - } finally { - await closeQuietly(store); - } - }); - - it('rejects a queued READ promptly when the caller aborts', async () => { - const store = makeStore({ operationTimeoutMs: 60_000 }); - try { - const busy = store.insert(quads(50_000)); - const controller = new AbortController(); - const read = store.query(BUSY_QUERY, { signal: controller.signal }); - controller.abort(new Error('caller aborted queued read')); - await expect(read).rejects.toThrow(/caller aborted queued read/); - await busy.catch(() => {}); - } finally { - await closeQuietly(store); - } - }); - - it('rejects when the caller aborts while registering the abort listener', async () => { - const store = makeStore({ operationTimeoutMs: 60_000 }); - try { - await expect( - store.query(BUSY_QUERY, { signal: abortDuringListenerRegistration('listener registration aborted') }), - ).rejects.toThrow(/listener registration aborted/); - } finally { - await closeQuietly(store); - } - }); - - it('does NOT bound mutations — a large insert under a tiny timeout still completes', async () => { - // The whole point of read-only scoping: a write is never reported as a clean - // failure while it's still in flight. With a 5ms bound a 50k insert (>>5ms) - // would reject if it were bounded; it must resolve instead. - const store = makeStore({ operationTimeoutMs: 5 }); - try { - await expect(store.insert(quads(50_000))).resolves.toBeUndefined(); - } finally { - await closeQuietly(store); - } - }); - - it('completes normally within a generous timeout', async () => { - const store = makeStore({ operationTimeoutMs: 60_000 }); - try { - await store.insert(quads(10)); - expect(await store.countQuads('urn:test:g')).toBe(10); - // The data lives in a NAMED graph, so the ASK must scope to it (a bare - // `ASK { ?s ?p ?o }` only matches the default graph). - const r = await store.query(BUSY_QUERY); - expect(r.type).toBe('boolean'); - if (r.type === 'boolean') expect(r.value).toBe(true); - } finally { - await closeQuietly(store); - } - }); - - it('operationTimeoutMs: 0 disables the timeout (a queued read still completes)', async () => { - const store = makeStore({ operationTimeoutMs: 0 }); - try { - const busy = store.insert(quads(50_000)); - // With the bound disabled, the read WAITS for the worker instead of - // rejecting, then returns true once the import lands. - const r = await store.query(BUSY_QUERY); - expect(r.type).toBe('boolean'); - if (r.type === 'boolean') expect(r.value).toBe(true); - await busy; - expect(await store.countQuads('urn:test:g')).toBe(50_000); - } finally { - await closeQuietly(store); - } - }); - - it('a fractional operationTimeoutMs is floored to an integer', async () => { - // normalizeNonNegativeInt floors the ms bound: 5.9 behaves like 5, so the - // surfaced error reports "after 5ms" (no fractional noise). - const store = makeStore({ operationTimeoutMs: 5.9 }); - try { - const busy = store.insert(quads(50_000)); - await expect(store.query(BUSY_QUERY)).rejects.toThrow(/timed out after 5ms/); - await busy.catch(() => {}); - } finally { - await closeQuietly(store); - } - }); - - it('close() is exempt from the per-op timeout so the final flush is never cut short', async () => { - // Codex review: close runs the worker's final flush; bounding it by the - // per-op timeout could terminate() the thread mid-flush and lose writes. - // With a tiny operationTimeoutMs and the worker still busy on a large - // insert, close() must WAIT for the worker to drain rather than reject — - // AND the in-flight write must actually land, not get killed mid-flush. - // We prove durability end-to-end on a persistent path: the regression this - // guards (close-after-debounced-flush race) was a SILENT data loss, so the - // test must assert the quads survive a reopen, not just that close resolves. - const dir = mkdtempSync(join(tmpdir(), 'oxigraph-worker-close-')); - const path = join(dir, 'store.nq'); - try { - const store = makeStore({ operationTimeoutMs: 10 }, path); - const busy = store.insert(quads(50_000)); // occupies the worker - // A read queued behind it times out at 10ms — proves the worker is busy. - await expect(store.query(BUSY_QUERY)).rejects.toThrow(/timed out/); - // close() must resolve cleanly (not reject with a 10ms timeout): it waits - // for the in-flight op to drain, then flushes + terminates. - await expect(store.close()).resolves.toBeUndefined(); - // The in-flight insert must have COMPLETED (drained), not been terminated - // mid-flight — otherwise close() silently cut it short. - await expect(busy).resolves.toBeUndefined(); - // And the write must be durable: a fresh worker on the same path hydrates - // all 50k quads, proving close() flushed before terminating. - const reopened = makeStore({ operationTimeoutMs: 60_000 }, path); - try { - expect(await reopened.countQuads('urn:test:g')).toBe(50_000); - } finally { - await closeQuietly(reopened); - } - } finally { - try { rmSync(dir, { recursive: true, force: true }); } catch { /* */ } - } - }); - - it('concurrent and repeated close() calls all resolve without hanging', async () => { - // Codex review: close() goes through an UNBOUNDED worker RPC, so a second - // close racing the first was orphaned when terminate() killed the worker and - // never settled. close() is now memoized + the worker 'exit' handler rejects - // anything still pending, so concurrent/repeat closes all settle. - const store = makeStore({ operationTimeoutMs: 60_000 }); - await store.insert(quads(5)); - const results = await Promise.all([store.close(), store.close(), store.close()]); - expect(results).toEqual([undefined, undefined, undefined]); - // Ops issued after close fail fast (store closed) instead of hanging. - await new Promise((r) => setImmediate(r)); - await expect(store.insert(quads(1))).rejects.toThrow(/closed/i); - }); - - it('store.options reach the adapter through createTripleStore (factory path)', async () => { - // Codex review: the user-facing path is createTripleStore({ backend, options }), - // not the constructor — assert the option forwarding in the adapter factory - // actually takes effect so a typo there can't silently drop the knob. - // operationTimeoutMs forwarded: a 5ms bound rejects a read queued behind a - // busy worker. - const store = await createTripleStore({ - backend: 'oxigraph-worker', - options: { operationTimeoutMs: 5 }, - }); - try { - const busy = store.insert(quads(50_000)); - await expect(store.query(BUSY_QUERY)).rejects.toThrow(/timed out after 5ms/); - await busy.catch(() => {}); - } finally { - await store.close().catch(() => {}); - } - }); -}); diff --git a/packages/storage/test/oxigraph-worker-respawn.test.ts b/packages/storage/test/oxigraph-worker-respawn.test.ts deleted file mode 100644 index 6d60206177..0000000000 --- a/packages/storage/test/oxigraph-worker-respawn.test.ts +++ /dev/null @@ -1,348 +0,0 @@ -import { describe, it, expect, beforeEach, afterEach } from 'vitest'; -import { mkdtempSync, rmSync } from 'node:fs'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import type { Worker } from 'node:worker_threads'; -import { OxigraphWorkerStore, type Quad } from '../src/index.js'; - -// Regression tests for unexpected-worker-exit recovery. The production -// failure: ERR_WORKER_OUT_OF_MEMORY kills ONLY the worker thread, the daemon -// process keeps running, and the old adapter latched `workerExited` forever — -// a testnet node served HTTP for DAYS with a dead store (green /api/status, -// 503 on every write). The fix auto-respawns the worker on an exit that was -// NOT initiated by close(); these tests kill the REAL worker thread (no -// mocks) by reaching into the private `worker` field and calling terminate(), -// which raises the exact same 'exit'-without-close signal as an OOM kill. -// -// Like the resilience suite next door, this needs the compiled worker -// artifact (`dist/adapters/oxigraph-worker-impl.js`); if it's missing we fail -// loudly with the remediation hint rather than silently skip. -function makeStore(persistPath?: string, opts?: { operationTimeoutMs?: number }): OxigraphWorkerStore { - try { - return new OxigraphWorkerStore(persistPath, opts); - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - if (/oxigraph-worker-impl/.test(msg)) { - throw new Error( - `oxigraph-worker adapter is not runnable — run ` + - `\`pnpm --filter @origintrail-official/dkg-storage build\` first. Underlying: ${msg}`, - ); - } - throw err; - } -} - -// The recovery machinery is intentionally private (nothing outside the -// adapter should steer it), so the tests reach in through one typed seam -// instead of scattering `as any` casts around. `lifecycle` is the explicit -// state field the respawn/close/in-memory-loss logic keys off (see the -// WorkerLifecycle model in the adapter); asserting on it pins the state model. -type WorkerLifecycle = - | 'initializing' - | 'live' - | 'respawning' - | 'closing' - | 'closed' - | 'gave_up' - | 'in_memory_lost'; -function internals(store: OxigraphWorkerStore): { - worker: Worker; - lifecycle: WorkerLifecycle; - closePromise: Promise | null; - respawnPromise: Promise | null; - consecutiveRespawnFailures: number; -} { - return store as unknown as { - worker: Worker; - lifecycle: WorkerLifecycle; - closePromise: Promise | null; - respawnPromise: Promise | null; - consecutiveRespawnFailures: number; - }; -} - -/** - * Simulate an OOM-style death of the CURRENT worker thread: terminate() while - * closePromise is null fires the same 'exit' event a crashed thread does. - * Awaiting terminate() guarantees the store's own 'exit' handler already ran - * (it was registered first, and 'exit' listeners run synchronously at emit), - * so on return the respawn has been scheduled — or already completed, for the - * immediate first attempt. - */ -async function killWorker(store: OxigraphWorkerStore): Promise { - await internals(store).worker.terminate(); -} - -function quads(n: number, graph = 'urn:test:g'): Quad[] { - const out: Quad[] = new Array(n); - for (let i = 0; i < n; i += 1) { - out[i] = { - subject: `urn:test:s:${i}`, - predicate: 'http://schema.org/name', - object: `"v${i}"`, - graph, - }; - } - return out; -} - -describe('OxigraphWorkerStore respawn after unexpected worker exit', () => { - let dir: string; - let path: string; - - beforeEach(() => { - dir = mkdtempSync(join(tmpdir(), 'oxigraph-worker-respawn-')); - path = join(dir, 'store.nq'); - }); - - afterEach(() => { - try { rmSync(dir, { recursive: true, force: true }); } catch { /* */ } - }); - - it('recovers from an unexpected worker death — subsequent ops succeed on a respawned worker', async () => { - const store = makeStore(path); - try { - await store.insert(quads(10)); - // insert() only schedules the worker's 50ms debounced flush; force the - // data to disk so the respawned worker hydrates it back. - await store.flush(); - - const before = internals(store).worker; - await killWorker(store); - - // Same store OBJECT (every daemon alias keeps working), new thread. - expect(internals(store).worker).not.toBe(before); - - // Reads see the persisted data again — the exact op that used to fail - // forever with "the store is closed". - expect(await store.countQuads('urn:test:g')).toBe(10); - // A disk-persisted store recovers fully: the state model is back to 'live' - // (this is the branch finding 1 deliberately keeps auto-respawning). - expect(internals(store).lifecycle).toBe('live'); - // And writes work too: the store is fully live, not read-only. - await store.insert(quads(5, 'urn:test:g2')); - expect(await store.countQuads('urn:test:g2')).toBe(5); - } finally { - await store.close().catch(() => {}); - } - }); - - it('rejects an in-flight op at kill time with the retryable "restarted — retry" message', async () => { - const store = makeStore(path); - try { - // A 50k insert occupies the single worker thread for hundreds of ms, so - // terminating right after posting is guaranteed to catch it in flight. - const inflight = store.insert(quads(50_000)); - await killWorker(store); - // The outcome of the killed op is genuinely unknown (the thread died - // mid-processing), so the caller gets a RETRYABLE error, not "closed". - await expect(inflight).rejects.toThrow(/oxigraph-worker restarted — retry/); - // ...and a retry against the same store object then succeeds. - await expect(store.insert(quads(10))).resolves.toBeUndefined(); - expect(await store.countQuads('urn:test:g')).toBe(10); - } finally { - await store.close().catch(() => {}); - } - }); - - it('parks ops issued during a respawn backoff instead of failing them', async () => { - const store = makeStore(path); - try { - await store.insert(quads(10)); - await store.flush(); - - // First kill → immediate respawn (backoff tier 0). Kill the replacement - // BEFORE it serves any op, so the second respawn sits in the 1s backoff - // tier — that's the window where new ops must park, not fail. - await killWorker(store); - await killWorker(store); - expect(internals(store).respawnPromise).not.toBeNull(); - - // Issued mid-backoff: the op waits for the replacement worker and then - // succeeds against the disk-persisted state (slightly slower success, - // never a spurious "store is closed"). - expect(await store.countQuads('urn:test:g')).toBe(10); - expect(internals(store).respawnPromise).toBeNull(); - } finally { - await store.close().catch(() => {}); - } - }, 15_000); - - it('a successful op resets the crash-loop counter', async () => { - const store = makeStore(path); - try { - await store.insert(quads(3)); - await store.flush(); - await killWorker(store); - // The respawn consumed one attempt... - expect(internals(store).consecutiveRespawnFailures).toBe(1); - // ...and the first successful reply proves the worker healthy, ending - // the accounting window (occasional crashes days apart never accumulate - // toward the give-up bound). - expect(await store.countQuads('urn:test:g')).toBe(3); - expect(internals(store).consecutiveRespawnFailures).toBe(0); - } finally { - await store.close().catch(() => {}); - } - }); - - it('close() does NOT respawn — the exit stays intentional and ops fail closed', async () => { - const store = makeStore(path); - await store.insert(quads(5)); - const lastWorker = internals(store).worker; - await store.close(); - - // Give any (buggy) respawn scheduling a chance to run before asserting. - await new Promise((r) => setTimeout(r, 50)); - expect(internals(store).respawnPromise).toBeNull(); - expect(internals(store).worker).toBe(lastWorker); - - // Post-close ops fail exactly as before this fix — fast and permanent. - await expect(store.countQuads('urn:test:g')).rejects.toThrow(/store is closed/); - await expect(store.insert(quads(1))).rejects.toThrow(/store is closed/); - }); - - it('gives up after MAX consecutive dead-on-arrival respawns and latches closed with guidance', async () => { - const store = makeStore(path); - try { - await store.insert(quads(2)); - await store.flush(); - // Simulate a genuine crash loop without waiting out the real 1s/5s/30s - // backoff ladder: pre-load the consecutive-failure counter to the bound, - // so the very next unexpected exit hits the give-up branch immediately. - internals(store).consecutiveRespawnFailures = 5; - await killWorker(store); - - // Latched closed — same fail-fast as the pre-recovery behaviour, but the - // error now tells the operator WHY and what to do about it. - const err: any = await store.countQuads('urn:test:g').then(() => null, (e) => e); - expect(err).toBeTruthy(); - expect(err.message).toMatch(/store is closed/); - expect(err.message).toMatch(/respawn gave up/); - // No further respawns get armed by later traffic. - expect(internals(store).respawnPromise).toBeNull(); - // The give-up verdict is now the single, explicit terminal state — no - // cluster of booleans that could disagree with the fail-fast error above. - expect(internals(store).lifecycle).toBe('gave_up'); - } finally { - // close() on a latched store must still resolve (idempotent teardown). - await expect(store.close()).resolves.toBeUndefined(); - } - }); - - it('close() during a respawn backoff wins — the store stays closed', async () => { - const store = makeStore(path); - await store.insert(quads(4)); - await store.flush(); - // Double-kill parks the second respawn in the 1s backoff tier (as above). - await killWorker(store); - await killWorker(store); - expect(internals(store).respawnPromise).not.toBeNull(); - - // An operator close arriving mid-backoff must never be resurrected by the - // supervisor: the pending respawn bails out when it wakes. - await expect(store.close()).resolves.toBeUndefined(); - // Wait out the backoff so a buggy respawn would have fired by now. - await new Promise((r) => setTimeout(r, 1_500)); - await expect(store.countQuads('urn:test:g')).rejects.toThrow(/store is closed/); - }, 15_000); -}); - -// Finding 1 (🔴): an IN-MEMORY store (constructed with NO persistPath) has no -// disk to reload from, so respawning after an unexpected worker exit would come -// up EMPTY yet look perfectly healthy — every row written before the crash -// silently gone (confirmed live: pre-crash data vanished). The fix is to FAIL -// CLOSED on such a store instead of respawning: the worker's data is -// unrecoverable, so subsequent ops must reject with a clear data-loss error, -// never return a wrong-but-plausible empty result. These tests use the REAL -// worker thread (no mocks) exactly like the persistent-store suite above, -// killing it via terminate() to raise the same 'exit'-without-close signal an -// OOM kill would. -describe('OxigraphWorkerStore in-memory fail-closed on unexpected worker exit', () => { - it('fails closed with a data-loss error after an in-memory worker crash — never a silent empty result', async () => { - // No persistPath → in-memory store. This is the exact shape that silently - // lost data before the fix. - const store = makeStore(undefined); - try { - await store.insert(quads(7)); - // Sanity: the data really is there before the crash. - expect(await store.countQuads('urn:test:g')).toBe(7); - - const before = internals(store).worker; - await killWorker(store); - - // The store must NOT respawn (no disk to recover from) — it latches into - // the terminal in_memory_lost state and never arms a replacement worker. - expect(internals(store).lifecycle).toBe('in_memory_lost'); - expect(internals(store).respawnPromise).toBeNull(); - expect(internals(store).worker).toBe(before); // no fresh (empty) thread - - // The regression this guards: a read here used to RESOLVE with 0 (an - // empty respawned store), reporting SUCCESS while all 7 rows were gone. - // `.then(() => null, e => e)` captures a resolve as null and a reject as - // the error, so a truthy `err` proves the read rejected rather than - // silently returning the wrong count. - const err: any = await store.countQuads('urn:test:g').then(() => null, (e) => e); - expect(err).toBeTruthy(); - expect(err.message).toMatch(/IN-MEMORY store's worker crashed/); - expect(err.message).toMatch(/data was lost|cannot be recovered/); - - // Writes fail closed the same way — the store is unusable, not read-only. - await expect(store.insert(quads(1))).rejects.toThrow(/IN-MEMORY store's worker crashed/); - } finally { - // close() on a data-lost store must still resolve (idempotent teardown) - // and must not resurrect it. - await expect(store.close()).resolves.toBeUndefined(); - } - }); - - it('the in_memory_lost state is terminal — later traffic never arms a respawn', async () => { - const store = makeStore(undefined); - try { - await store.insert(quads(3)); - await killWorker(store); - expect(internals(store).lifecycle).toBe('in_memory_lost'); - - // Hammer it with a few more ops: each must fail fast, and none may flip - // the store back to 'respawning'/'live' (the terminal-state guarantee). - for (let i = 0; i < 3; i += 1) { - await expect(store.countQuads('urn:test:g')).rejects.toThrow(/cannot be recovered/); - expect(internals(store).lifecycle).toBe('in_memory_lost'); - expect(internals(store).respawnPromise).toBeNull(); - } - } finally { - await store.close().catch(() => {}); - } - }); - - it('otReviewAgent #1408: the spawn→live transition is guarded — a spawn from a terminal state THROWS (never resurrects the store), but the two legal predecessors enter live', async () => { - // The spawn path (spawnWorker → markSpawnedLive) is the one writer that - // enters 'live'. It must still enforce terminal permanence: were a future - // respawn/close code path to call it after close/give-up/in-memory-loss, it - // has to throw rather than silently reopen the store. Poke the state field - // directly (decoupled from the real worker) to drive the guard. - const store = makeStore(undefined); - const seam = store as unknown as { - lifecycle: WorkerLifecycle; - markSpawnedLive: () => void; - }; - try { - for (const terminal of ['closed', 'gave_up', 'in_memory_lost'] as const) { - seam.lifecycle = terminal; - expect(() => seam.markSpawnedLive()).toThrow(/illegal spawn transition|terminal/i); - expect(seam.lifecycle).toBe(terminal); // stayed terminal — not resurrected - } - // 'closing' is likewise not a legal spawn predecessor. - seam.lifecycle = 'closing'; - expect(() => seam.markSpawnedLive()).toThrow(/illegal spawn transition/i); - // The two legal predecessors DO enter 'live'. - for (const start of ['initializing', 'respawning'] as const) { - seam.lifecycle = start; - expect(() => seam.markSpawnedLive()).not.toThrow(); - expect(seam.lifecycle).toBe('live'); - } - } finally { - await store.close().catch(() => {}); - } - }); -}); diff --git a/packages/storage/test/storage.test.ts b/packages/storage/test/storage.test.ts index d9188f06f2..8ac362dca2 100644 --- a/packages/storage/test/storage.test.ts +++ b/packages/storage/test/storage.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach, beforeAll, afterAll } from 'vitest'; +import { describe, it, expect, expectTypeOf, beforeEach, beforeAll, afterAll } from 'vitest'; import { OxigraphStore, BlazegraphStore, @@ -6,6 +6,8 @@ import { GraphManager, PrivateContentStore, createTripleStore, + classifyTripleStoreBackend, + customTripleStoreBackend, loadSelectedSharedMemoryQuads, loadSelectedVerifiableMemoryQuads, registerTripleStoreAdapter, @@ -13,6 +15,7 @@ import { resolveVerifiableMemoryReadGraphs, type Quad, type TripleStore, + type TripleStoreBackend, } from '../src/index.js'; import { contextGraphDataGraphUri, @@ -252,12 +255,17 @@ if (blazeUrl) { // --------------------------------------------------------------------------- describe('createTripleStore factory', () => { + it('keeps managed and retired daemon names out of the constructible backend type', () => { + expectTypeOf<'oxigraph'>().toMatchTypeOf(); + expectTypeOf<'oxigraph-server'>().not.toMatchTypeOf(); + expectTypeOf<'oxigraph-worker'>().not.toMatchTypeOf(); + }); + it('all built-in backends are registered (factory throws something other than "Unknown TripleStore backend")', async () => { // The *registry* contract being tested here is: every built-in // backend name is recognized. The construction itself may require - // options (blazegraph needs `url`, sparql-http needs `queryEndpoint`) - // or worker artifacts (oxigraph-worker needs the compiled worker - // impl). So a backend passes this test iff calling `createTripleStore` + // options (blazegraph needs `url`, sparql-http needs `queryEndpoint`). + // So a backend passes this test iff calling `createTripleStore` // either succeeds OR throws a *non*-"Unknown TripleStore backend" // error. // @@ -266,7 +274,7 @@ describe('createTripleStore factory', () => { // effectively assert "a promise settled" — noise. This version // asserts the positive contract explicitly and points at the // specific failing backend if the registry regresses. - const backends = ['oxigraph', 'oxigraph-worker', 'blazegraph', 'sparql-http']; + const backends = ['oxigraph', 'blazegraph', 'sparql-http']; for (const backend of backends) { let outcome: 'constructed' | Error; try { @@ -312,56 +320,43 @@ describe('createTripleStore factory', () => { ).rejects.toThrow('queryEndpoint'); }); - it('oxigraph-worker adapter is registered and round-trips an insert', async () => { - // The worker adapter resolves `./oxigraph-worker-impl.js` relative to - // the module loaded at runtime. When vitest runs against raw source - // without a prior `pnpm build`, that URL lands in `src/adapters/` where - // only the .ts files live, so the Worker constructor throws - // "Cannot find module … oxigraph-worker-impl.js". - // - // This used to be caught and converted to `ctx.skip()`, which meant a - // green CI run even when the worker artifact was missing — i.e. a - // broken build never triggered a test failure. We now FAIL LOUDLY in - // that case with a remediation hint, so: - // • locally, the developer sees "run pnpm build first" instead of a - // silent skip; - // • in CI, if `pnpm build` was not wired into the lane (or the build - // regresses), this test surfaces it as a red failure. - let store: Awaited>; - try { - store = await createTripleStore({ backend: 'oxigraph-worker' }); - } catch (err: unknown) { - const msg = err instanceof Error ? err.message : String(err); - if (msg.includes('Cannot find module') && msg.includes('oxigraph-worker-impl')) { - throw new Error( - `oxigraph-worker adapter is not runnable — the compiled ` + - `oxigraph-worker-impl.js artifact is missing from ` + - `packages/storage/dist/adapters/. Run ` + - `\`pnpm --filter @origintrail-official/dkg-storage build\` ` + - `before running this test. Underlying error: ${msg}`, - ); - } - throw err; - } - await store.insert([{ - subject: 'http://ex.org/s', - predicate: 'http://ex.org/p', - object: '"hi"', - graph: 'http://ex.org/g', - }]); - expect(await store.countQuads()).toBe(1); - await store.close(); - }); - it('throws on unknown backend', async () => { - await expect(createTripleStore({ backend: 'unknown' })).rejects.toThrow( + await expect(createTripleStore({ backend: customTripleStoreBackend('unknown') })).rejects.toThrow( 'Unknown TripleStore backend', ); }); + it('gives runtime migration guidance to legacy oxigraph-worker callers', async () => { + await expect(createTripleStore({ + backend: 'oxigraph-worker' as any, + })).rejects.toThrow( + /oxigraph-worker.*no longer supported.*sparql-http.*oxigraph-persistent/, + ); + }); + + it('classifies only factory adapters and leaves daemon policy names outside storage', async () => { + expect(classifyTripleStoreBackend('oxigraph')).toEqual({ + kind: 'adapter', + backend: 'oxigraph', + }); + expect(classifyTripleStoreBackend('oxigraph-server')).toEqual({ + kind: 'custom', + backend: 'oxigraph-server', + }); + expect(classifyTripleStoreBackend('oxigraph-worker')).toEqual({ + kind: 'custom', + backend: 'oxigraph-worker', + }); + await expect(createTripleStore({ + backend: customTripleStoreBackend('oxigraph-server'), + })).rejects.toThrow( + /Unknown TripleStore backend/, + ); + }); + it('custom adapter can be registered and used', async () => { const calls: string[] = []; - registerTripleStoreAdapter('test-custom', async () => ({ + const backend = registerTripleStoreAdapter('test-custom', async () => ({ insert: async () => { calls.push('insert'); }, delete: async () => {}, deleteByPattern: async () => 0, @@ -375,7 +370,7 @@ describe('createTripleStore factory', () => { close: async () => {}, })); - const store = await createTripleStore({ backend: 'test-custom' }); + const store = await createTripleStore({ backend }); await store.insert([]); expect(calls).toEqual(['insert']); expect(await store.countQuads()).toBe(1); diff --git a/scripts/devnet.sh b/scripts/devnet.sh index ff21784c35..03fabea9a3 100755 --- a/scripts/devnet.sh +++ b/scripts/devnet.sh @@ -546,7 +546,7 @@ create_node_config() { # and spawns it on its own port — NO Docker. Node 1 is the node the # UI/e2e suite drives, so the suite now exercises the REAL default # backend and deterministically reproduces SPARQL-over-HTTP-only - # bugs such as #996 — which the old `oxigraph-worker` default hid.) + # bugs such as #996.) # Node 3-4: blazegraph (if Docker) else oxigraph (in-process baseline) # Node 5-6: sparql-http → external Dockerized Oxigraph (EXTRA coverage of the # generic external-endpoint path; Docker-only, optional) @@ -573,6 +573,8 @@ create_node_config() { local ox_port_var="OXIGRAPH_SERVER_PORT_${node_num}" local ox_port="${!ox_port_var}" store_block="\"store\": { \"backend\": \"sparql-http\", \"options\": { \"queryEndpoint\": \"http://127.0.0.1:${ox_port}/query\", \"updateEndpoint\": \"http://127.0.0.1:${ox_port}/update\" } }," + else + store_block="\"store\": { \"backend\": \"oxigraph\" }," fi fi @@ -1769,12 +1771,12 @@ cmd_start() { local api_port=$((API_PORT_BASE + i - 1)) local role="edge" [ "$i" -le "$NUM_CORE_NODES" ] && role="core" - local store_label="oxigraph-worker" + local store_label="oxigraph-server" if [ "$i" -ge 3 ] && [ "$i" -le 4 ]; then [ "$BLAZEGRAPH_AVAILABLE" = true ] && store_label="blazegraph" || store_label="oxigraph" fi if [ "$i" -ge 5 ]; then - [ "$OXIGRAPH_SERVER_AVAILABLE" = true ] && store_label="oxigraph-server" || store_label="oxigraph-worker" + [ "$OXIGRAPH_SERVER_AVAILABLE" = true ] && store_label="sparql-http" || store_label="oxigraph" fi log "Node $i ($role, $store_label): http://127.0.0.1:$api_port/ui" done diff --git a/scripts/publisher-smoke-test.sh b/scripts/publisher-smoke-test.sh index 1086fe0cc5..28472b5660 100755 --- a/scripts/publisher-smoke-test.sh +++ b/scripts/publisher-smoke-test.sh @@ -76,7 +76,7 @@ import { DKGPublisher, TripleStoreAsyncLiftPublisher } from '@origintrail-offici const dkgHome = process.env.DKG_HOME; const privateKey = process.env.SMOKE_PRIVATE_KEY; const store = await createTripleStore({ - backend: 'oxigraph-worker', + backend: 'oxigraph-persistent', options: { path: join(dkgHome, 'store.nq') }, });