Merge pull request #217 from InsForge/feat/webscraper-self-hosted-connect

feat(webscraper): connect Apify with an API token on self-hosted backends

Author: CarmenDou

src/commands/webscraper/apify/connect.test.ts

@@ -0,0 +1,269 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { Command } from 'commander';
+
+const apiMock = vi.hoisted(() => ({
+  startApifyCliFlow: vi.fn(),
+  pollApifyConnection: vi.fn(),
+  fetchApifyConnection: vi.fn(),
+}));
+const apifyConfigMock = vi.hoisted(() => ({
+  storeApifyToken: vi.fn(),
+}));
+vi.mock('../../../lib/api/webscraper.js', () => ({ ...apiMock, ...apifyConfigMock }));
+
+const configMock = vi.hoisted(() => ({
+  getProjectConfig: vi.fn(() => ({ project_id: 'p1', project_name: 'Test Project' })),
+  getAccessToken: vi.fn((): string | null => 'tok'),
+}));
+vi.mock('../../../lib/config.js', () => configMock);
+
+const analyticsMock = vi.hoisted(() => ({
+  trackGroupCommand: vi.fn(),
+  shutdownAnalytics: vi.fn(async () => {}),
+}));
+vi.mock('../../../lib/analytics.js', () => analyticsMock);
+
+const bridgeMock = vi.hoisted(() => ({
+  runApifyAuthBridge: vi.fn(async () => ({ skillsInstalled: true })),
+}));
+vi.mock('../../../lib/apify-bridge.js', () => bridgeMock);
+
+vi.mock('../../../lib/prompts.js', () => ({ isInteractive: false }));
+
+// `open` is loaded dynamically inside runConnectFlow (OAuth path); mock so the
+// real browser launch doesn't fire during tests.
+vi.mock('open', () => ({ default: vi.fn() }));
+
+// Silence interactive UI noise from clack — tests assert on mocks, not stdout.
+vi.mock('@clack/prompts', async (orig) => {
+  const actual = (await orig()) as Record<string, unknown>;
+  return {
+    ...actual,
+    intro: vi.fn(),
+    outro: vi.fn(),
+    log: { info: vi.fn(), success: vi.fn(), warn: vi.fn(), error: vi.fn() },
+    spinner: vi.fn(() => ({ start: vi.fn(), stop: vi.fn(), message: vi.fn() })),
+  };
+});
+
+const outputMock = vi.hoisted(() => ({
+  outputJson: vi.fn(),
+  outputSuccess: vi.fn(),
+}));
+vi.mock('../../../lib/output.js', () => outputMock);
+
+// Imports must come AFTER the vi.mock calls because Vitest hoists the mocks
+// but ESM module evaluation order still matters.
+import { registerApifyConnectCommand } from './connect.js';
+import { CLIError } from '../../../lib/errors.js';
+
+interface RunResult {
+  exitCode?: number;
+}
+
+// Override process.exit so handleError doesn't kill the test process; capture
+// the first exit code. Mirrors src/commands/posthog/setup.test.ts.
+async function runConnect(argv: string[]): Promise<RunResult> {
+  const program = new Command();
+  program.option('--json').option('--api-url <url>').option('-y, --yes');
+  const webscraper = program.command('webscraper');
+  const apify = webscraper.command('apify');
+  registerApifyConnectCommand(apify);
+
+  const origExit = process.exit;
+  const result: RunResult = {};
+  (process.exit as unknown) = (code?: number) => {
+    if (result.exitCode === undefined) result.exitCode = code;
+    throw new Error('__exit__');
+  };
+  try {
+    await program
+      .parseAsync(['node', 'test', 'webscraper', 'apify', 'connect', ...argv])
+      .catch((err) => {
+        if (err instanceof Error && err.message === '__exit__') return;
+        throw err;
+      });
+  } finally {
+    process.exit = origExit;
+  }
+  return result;
+}
+
+beforeEach(() => {
+  apiMock.startApifyCliFlow.mockReset();
+  apiMock.pollApifyConnection.mockReset();
+  apiMock.fetchApifyConnection.mockReset();
+  apifyConfigMock.storeApifyToken.mockReset();
+  bridgeMock.runApifyAuthBridge.mockReset().mockResolvedValue({ skillsInstalled: true });
+  outputMock.outputJson.mockReset();
+  outputMock.outputSuccess.mockReset();
+  analyticsMock.trackGroupCommand.mockReset();
+  analyticsMock.shutdownAnalytics.mockClear();
+  configMock.getProjectConfig.mockReturnValue({ project_id: 'p1', project_name: 'Test Project' });
+  configMock.getAccessToken.mockReturnValue('tok');
+});
+
+describe('apify connect', () => {
+  describe('--token path (self-hosted)', () => {
+    it('stores the token and skips OAuth entirely', async () => {
+      apifyConfigMock.storeApifyToken.mockResolvedValue({
+        configured: true,
+        maskedKey: 'apify_ap••••••••mnop',
+      });
+
+      await runConnect(['--token', 'apify_api_tok1234567890']);
+
+      expect(apifyConfigMock.storeApifyToken).toHaveBeenCalledWith('apify_api_tok1234567890');
+      expect(apiMock.startApifyCliFlow).not.toHaveBeenCalled();
+      expect(apiMock.pollApifyConnection).not.toHaveBeenCalled();
+      expect(apiMock.fetchApifyConnection).not.toHaveBeenCalled();
+      expect(bridgeMock.runApifyAuthBridge).not.toHaveBeenCalled();
+    });
+
+    it('does not require a login token', async () => {
+      configMock.getAccessToken.mockReturnValue(null);
+      apifyConfigMock.storeApifyToken.mockResolvedValue({
+        configured: true,
+        maskedKey: 'apify_ap••••••••mnop',
+      });
+
+      const r = await runConnect(['--token', 'apify_api_tok1234567890']);
+
+      expect(r.exitCode).toBeUndefined();
+      expect(apifyConfigMock.storeApifyToken).toHaveBeenCalledOnce();
+    });
+
+    it('prints the masked key, never the raw token, in non-json mode', async () => {
+      apifyConfigMock.storeApifyToken.mockResolvedValue({
+        configured: true,
+        maskedKey: 'apify_ap••••••••mnop',
+      });
+
+      await runConnect(['--token', 'apify_api_tok1234567890']);
+
+      expect(outputMock.outputSuccess).toHaveBeenCalledWith(
+        expect.stringContaining('apify_ap••••••••mnop'),
+      );
+      const allCalls = [...outputMock.outputSuccess.mock.calls, ...outputMock.outputJson.mock.calls];
+      const serialized = JSON.stringify(allCalls);
+      expect(serialized).not.toContain('apify_api_tok1234567890');
+    });
+
+    it('emits the masked token status in --json mode', async () => {
+      apifyConfigMock.storeApifyToken.mockResolvedValue({
+        configured: true,
+        maskedKey: 'apify_ap••••••••mnop',
+      });
+
+      await runConnect(['--json', '--token', 'apify_api_tok1234567890']);
+
+      expect(outputMock.outputJson).toHaveBeenCalledOnce();
+      const payload = outputMock.outputJson.mock.calls[0][0] as {
+        success: boolean;
+        connectionState: string;
+        token: { configured: boolean; maskedKey: string | null };
+      };
+      expect(payload.success).toBe(true);
+      expect(payload.connectionState).toBe('newly-connected');
+      expect(payload.token).toEqual({ configured: true, maskedKey: 'apify_ap••••••••mnop' });
+    });
+
+    it('propagates a rejected token as a CLI error and exits non-zero', async () => {
+      apifyConfigMock.storeApifyToken.mockRejectedValue(
+        new CLIError('Apify rejected this API token.', 1, 'INVALID_INPUT', 400),
+      );
+
+      const r = await runConnect(['--token', 'bogus']);
+
+      expect(r.exitCode).toBe(1);
+    });
+
+    it('rejects an empty --token locally instead of falling through to OAuth', async () => {
+      const r = await runConnect(['--token', '']);
+
+      expect(r.exitCode).toBe(1);
+      expect(apifyConfigMock.storeApifyToken).not.toHaveBeenCalled();
+      expect(apiMock.startApifyCliFlow).not.toHaveBeenCalled();
+      expect(apiMock.pollApifyConnection).not.toHaveBeenCalled();
+      expect(apiMock.fetchApifyConnection).not.toHaveBeenCalled();
+      expect(bridgeMock.runApifyAuthBridge).not.toHaveBeenCalled();
+    });
+
+    it('rejects a whitespace-only --token locally instead of falling through to OAuth', async () => {
+      const r = await runConnect(['--token', '   ']);
+
+      expect(r.exitCode).toBe(1);
+      expect(apifyConfigMock.storeApifyToken).not.toHaveBeenCalled();
+      expect(apiMock.startApifyCliFlow).not.toHaveBeenCalled();
+      expect(bridgeMock.runApifyAuthBridge).not.toHaveBeenCalled();
+    });
+  });
+
+  describe('OAuth path (--token absent) is unchanged', () => {
+    it('fast path: cli-start says connected → verifies via /connection, skips polling, runs the auth bridge', async () => {
+      apiMock.startApifyCliFlow.mockResolvedValue({ type: 'connected' });
+      apiMock.fetchApifyConnection.mockResolvedValue({
+        kind: 'connected',
+        connection: { apifyUsername: 'someone', plan: 'free', status: 'active' },
+      });
+
+      await runConnect(['--skip-browser']);
+
+      expect(apiMock.startApifyCliFlow).toHaveBeenCalledOnce();
+      expect(apiMock.fetchApifyConnection).toHaveBeenCalledOnce();
+      expect(apiMock.pollApifyConnection).not.toHaveBeenCalled();
+      expect(bridgeMock.runApifyAuthBridge).toHaveBeenCalledOnce();
+      expect(apifyConfigMock.storeApifyToken).not.toHaveBeenCalled();
+    });
+
+    it('slow path: cli-start returns authorizeUrl → polls until connected', async () => {
+      apiMock.startApifyCliFlow.mockResolvedValue({
+        type: 'authorize',
+        authorizeUrl: 'https://example.com/auth',
+      });
+      apiMock.pollApifyConnection.mockResolvedValue({
+        apifyUsername: 'someone',
+        plan: 'free',
+        status: 'active',
+      });
+
+      await runConnect(['--skip-browser']);
+
+      expect(apiMock.pollApifyConnection).toHaveBeenCalledOnce();
+      expect(apiMock.fetchApifyConnection).not.toHaveBeenCalled();
+      expect(apifyConfigMock.storeApifyToken).not.toHaveBeenCalled();
+    });
+
+    it('still requires a login token, exactly as before', async () => {
+      configMock.getAccessToken.mockReturnValue(null);
+      apiMock.startApifyCliFlow.mockResolvedValue({ type: 'connected' });
+
+      const r = await runConnect(['--skip-browser']);
+
+      expect(r.exitCode).toBeGreaterThan(0);
+      expect(apiMock.startApifyCliFlow).not.toHaveBeenCalled();
+    });
+
+    it('emits JSON with connectionState and connection, no token field', async () => {
+      apiMock.startApifyCliFlow.mockResolvedValue({ type: 'connected' });
+      apiMock.fetchApifyConnection.mockResolvedValue({
+        kind: 'connected',
+        connection: { apifyUsername: 'someone', plan: 'free', status: 'active' },
+      });
+
+      await runConnect(['--json', '--skip-browser']);
+
+      expect(outputMock.outputJson).toHaveBeenCalledOnce();
+      const payload = outputMock.outputJson.mock.calls[0][0] as {
+        success: boolean;
+        connectionState: string;
+        connection: unknown;
+        token?: unknown;
+      };
+      expect(payload.success).toBe(true);
+      expect(payload.connectionState).toBe('already-connected');
+      expect(payload.connection).toMatchObject({ apifyUsername: 'someone', plan: 'free' });
+      expect(payload.token).toBeUndefined();
+    });
+  });
+});

src/commands/webscraper/apify/connect.ts

@@ -14,8 +14,10 @@ import {
   fetchApifyConnection,
   pollApifyConnection,
   startApifyCliFlow,
+  storeApifyToken,
   type ApifyConnectionResponse,
-} from '../../../lib/api/apify.js';
+  type ApifyTokenStatus,
+} from '../../../lib/api/webscraper.js';
 import { outputJson, outputSuccess } from '../../../lib/output.js';
 import { trackGroupCommand, shutdownAnalytics } from '../../../lib/analytics.js';
 import { runApifyAuthBridge } from '../../../lib/apify-bridge.js';
@@ -33,20 +35,24 @@ interface ConnectResult {
     plan?: string | null;
     status?: string;
   };
+  /** Present only on the self-hosted `--token` path. */
+  token?: ApifyTokenStatus;
 }
 
 export function registerApifyConnectCommand(program: Command): void {
   program
     .command('connect')
     .description('Connect your Apify account to your InsForge project')
     .option('--skip-browser', 'Do not auto-open the browser for OAuth; only print the URL')
+    .option('--token <token>', 'Apify API token (self-hosted; skips the OAuth flow)')
     .action(async (opts, cmd) => {
       const { json, apiUrl } = getRootOpts(cmd);
       try {
         const result = await runConnect({
           json,
           apiUrl,
           skipBrowser: Boolean(opts.skipBrowser),
+          token: opts.token,
         });
         if (json) {
           outputJson({ success: true, ...result });
@@ -69,6 +75,8 @@ interface RunConnectOpts {
   json: boolean;
   apiUrl?: string;
   skipBrowser: boolean;
+  /** Self-hosted path: an Apify API token to store directly, bypassing OAuth. */
+  token?: string;
 }
 
 // Ensures the InsForge project has an Apify connection (cli-start / OAuth).
@@ -82,6 +90,33 @@ async function runConnect(opts: RunConnectOpts): Promise<ConnectResult> {
     throw new ProjectNotLinkedError();
   }
 
+  // Self-hosted token path: the token is validated and stored server-side via
+  // the existing admin-authenticated ossFetch, so it needs neither the login
+  // token below nor any of the OAuth / auth-bridge flow further down. Short-
+  // circuits before any OAuth work; everything below is unreachable and thus
+  // unchanged when --token is absent.
+  //
+  // Branch on the option being *supplied*, not on its truthiness: `--token ""`
+  // must not silently fall through to the OAuth flow below (which would print
+  // a confusing "not logged in" error in CI when an env var expands to empty).
+  // Reject it locally instead — the backend's zod message for this case
+  // ("String must contain at least 1 character(s)") is far less clear.
+  if (opts.token !== undefined) {
+    if (opts.token.trim().length === 0) {
+      throw new CLIError('--token requires a non-empty Apify API token.');
+    }
+    trackGroupCommand('apify', 'connect', proj);
+    const status = await storeApifyToken(opts.token);
+    if (!opts.json) {
+      outputSuccess(`Apify connected with token ${status.maskedKey ?? '(hidden)'}`);
+    }
+    return {
+      connectionState: 'newly-connected',
+      connection: { apifyUsername: null, plan: null, status: 'active' },
+      token: status,
+    };
+  }
+
   // 2. Login token
   const token = getAccessToken();
   if (!token) {

src/lib/api/oss.test.ts

@@ -76,4 +76,29 @@ describe('ossFetch', () => {
       /Upgrade your InsForge project to a version with Model Gateway support/,
     );
   });
+
+  it('shows an upgrade+token message for a route-level 404 under /api/webscraper (not a cloud-only warning)', async () => {
+    // A 404 here means the backend predates the web scraper feature, not that
+    // self-hosting is unsupported — self-hosted backends do serve this route
+    // once they're upgraded. Guards against regressing to the old wording.
+    vi.spyOn(config, 'getProjectConfig').mockReturnValue({
+      project_id: 'p1',
+      project_name: 'demo',
+      org_id: 'o1',
+      appkey: 'app',
+      region: 'us-east',
+      api_key: 'ik_test',
+      oss_host: 'https://app.us-east.insforge.app',
+    } satisfies ProjectConfig);
+    vi.spyOn(globalThis, 'fetch').mockResolvedValueOnce(
+      new Response(JSON.stringify({ error: 'NOT_FOUND' }), {
+        status: 404,
+        headers: { 'Content-Type': 'application/json' },
+      }),
+    );
+
+    await expect(ossFetch('/api/webscraper/apify/config')).rejects.toThrow(
+      /Upgrade your InsForge instance.*insforge webscraper apify connect --token/s,
+    );
+  });
 });

src/lib/api/oss.ts

@@ -210,7 +210,7 @@ export async function ossFetch(
     }
 
     if (res.status === 404 && isRouteLevel404 && path.startsWith('/api/webscraper')) {
-      message = 'The web scraper is not available on this backend.\nThe Apify web scraper is cloud-only. Self-hosted: this feature is not supported. Cloud: contact your InsForge admin to enable it.';
+      message = 'The web scraper is not available on this backend.\nUpgrade your InsForge instance to a version with web scraper support, then run `insforge webscraper apify connect --token <token>` to connect your Apify account.';
     }
 
     // Safe to treat any 404 on /api/advisor/* as a route-level miss: the OSS