diff --git a/.env.sample b/.env.sample index 4379d1208..28a0fb61a 100644 --- a/.env.sample +++ b/.env.sample @@ -7,6 +7,8 @@ GOOGLE_API_KEY=your-google-api-key-here # OPENAI_API_KEY=your-openai-api-key-here # OPENROUTER_API_KEY=your-openrouter-api-key-here +# MINIMAX_API_KEY=your-minimax-api-key-or-subscription-key-here +# MINIMAX_BASE_URL=https://api.minimax.io/v1 # === Optional: Azure OpenAI === # AZURE_OPENAI_API_KEY= diff --git a/.research/deep-research-deepwiki-open-sync-evaluation.md b/.research/deep-research-deepwiki-open-sync-evaluation.md new file mode 100644 index 000000000..e160c0a72 --- /dev/null +++ b/.research/deep-research-deepwiki-open-sync-evaluation.md @@ -0,0 +1,118 @@ +# Deep Research Report: DeepWiki Open Local Fork, Upstream Sync, and OpenDeepWiki Comparison + +**Generated**: 2026-07-01 +**Research Parameters**: Breadth 3, Depth 1 +**Sources Consulted**: local git/source, GitHub `gh` metadata/API, hosted DeepWiki MCP direct streamable HTTP probes, Codex-native explorer agents + +## Executive Summary + +The local `RichelynScott/deepwiki-open` fork is a focused self-hosted +DeepWiki-Open integration with a custom FastMCP stdio wrapper around a local +Dockerized FastAPI/Next.js app. Its highest-value customizations are local path +translation, stream aggregation, usage/cost telemetry, stale-cache warnings, and +ephemeral-file exclusion for RAG citation hygiene. + +AsyncFuncAI upstream has 8 commits after the local merge-base `05591ee`. The +sync is technically clean and low-conflict, adding LiteLLM support, Docker +tiktoken cache preloading, vendor-dir exclusion, and docs updates. It is not +security-clean for broad deployment: open public reports cover unauthenticated +file-read/git-clone SSRF, dependency advisories, and missing private disclosure +workflow. The sync branch is therefore suitable for review/local use, not for +network exposure without hardening. + +`AIDotNet/OpenDeepWiki` is a more complete self-hosted knowledge-base platform, +not a drop-in replacement. It has releases, database-backed admin/user/API-key +features, background workers, and built-in MCP endpoints, but also a larger +.NET/Next.js/database/runtime surface that should be evaluated in isolation. + +## Detailed Findings + +### Local Fork + +- Local checkout started clean on `main` and aligned with `origin/main`. +- The repo was initially shallow; `git fetch --unshallow origin` was required + before `git merge-base HEAD upstream/main` returned `05591ee`. +- Fork-owned seams are `mcp_wrapper.py`, `docker-compose.override.yml`, + `requirements-mcp.txt`, `.env.sample`, `CLAUDE.md`, `FYI.md`, and `docs/`. +- The no-delete static gate found `Path.unlink()` in `mcp_wrapper.py`; the sync + branch changes force-refresh cache invalidation to move old `.pkl` files into + `~/.adalflow/databases/archive/` instead. + +### AsyncFuncAI Upstream + +- `gh repo view AsyncFuncAI/deepwiki-open` reported MIT license, default branch + `main`, 17,113 stars, 1,922 forks, no latest release, pushed 2026-06-03. +- Hosted DeepWiki MCP became available after live Codex config repair. Direct + streamable HTTP probes confirmed server `DeepWiki` version `2.14.3`, listed + `read_wiki_structure`, `read_wiki_contents`, and `ask_question`, and returned + an AsyncFuncAI wiki structure covering overview, backend architecture, RAG + pipeline, frontend architecture, API reference, tests, and glossary. +- Hosted DeepWiki MCP corroborated that AsyncFuncAI is a FastAPI + Next.js RAG + app using local FAISS-style storage, repository cloning, configurable + Google/OpenAI/OpenRouter/AWS Bedrock/Ollama/Azure providers, permissive CORS, + and exposed backend/frontend ports. +- The merged upstream commits add LiteLLM files (`api/litellm_client.py`, + `Dockerfile-litellm`, `docker-compose-litellm.yml`, `litellm-config.yml`) and + tiktoken cache preload in `Dockerfile`. +- Open upstream issues include security and dependency reports, notably + unauthenticated file-read/git-clone SSRF and dependency advisories. +- `npm audit --audit-level=moderate --json` in the sync worktree reported 23 + vulnerabilities: 17 moderate, 5 high, and 1 critical. + +### AIDotNet/OpenDeepWiki + +- `gh repo view AIDotNet/OpenDeepWiki` reported MIT license, default branch + `main`, latest release `v2.0.3` published 2026-05-30, pushed 2026-06-13, + 3,376 stars, and 432 forks. +- Hosted DeepWiki MCP returned an AIDotNet wiki structure covering architecture, + AI document generation, repository management, chat assistant, MCP, admin UI, + backend API/services, infrastructure, and development guide. +- Hosted DeepWiki MCP corroborated that AIDotNet is a .NET 10 + Next.js 16 + platform with Docker Compose install, AI provider/model binding, Git/ZIP/local + ingestion, MCP endpoints, permissive CORS, local path root allowlisting, and + plaintext `AuthPassword` repository credential storage. +- README/source inspection shows .NET 10 backend, Next.js 16/React 19 frontend, + SQLite/PostgreSQL, LibGit2Sharp repo processing, background workers, admin + console, users/roles/departments/API keys, chat, and MCP endpoints. +- It has better product completeness and release discipline than AsyncFuncAI, + but introduces heavier runtime and security review requirements. +- High-risk defaults or surfaces include default JWT/admin bootstrap, + permissive CORS, root compose runtime, plaintext private repo credential + fields, disabled Git TLS certificate checks, ZIP/local ingestion, and Graphify + subprocess execution. + +## Key Data Points + +| Item | Value | +|---|---| +| Local merge-base | `05591ee7160fcc8eeb4d976d10ce566a16c2eb94` | +| Local ahead / upstream behind | local 17 ahead, upstream 8 ahead before sync | +| Sync branch | `sync/upstream-20260701` | +| Sync commit | current branch tip; run `git rev-parse HEAD` | +| Sync worktree | `/home/riche/MCPs/deepwiki-open/WORKTREES/upstream-20260701` | +| AsyncFuncAI stars/forks | 17,113 / 1,922 | +| AIDotNet stars/forks | 3,376 / 432 | +| npm audit count | 23 total: 17 moderate, 5 high, 1 critical | + +## Emerging Trends And Contrarian Views + +- The local wrapper is becoming more valuable than the upstream app for agent + workflows because it adds operational safety and cost telemetry that upstream + does not provide. +- AIDotNet is attractive if the goal changes from single-user local RAG to + managed team knowledge-base service. For the current local MCP use case, its + complexity is a liability until isolated evaluation proves otherwise. +- Upstream parity is less important than hardening the ingestion boundary: + bind scope, auth, CORS, local path policy, SSRF defenses, dependency updates, + and token handling should drive the next branch. + +## Sources + +- Local repository: `/home/riche/MCPs/deepwiki-open` +- Sync worktree: `/home/riche/MCPs/deepwiki-open/WORKTREES/upstream-20260701` +- Hosted DeepWiki MCP: `https://mcp.deepwiki.com/mcp` +- AsyncFuncAI repo: https://github.com/AsyncFuncAI/deepwiki-open +- AsyncFuncAI issue list: https://github.com/AsyncFuncAI/deepwiki-open/issues +- AIDotNet repo: https://github.com/AIDotNet/OpenDeepWiki +- AIDotNet releases: https://github.com/AIDotNet/OpenDeepWiki/releases +- Review report: `docs/DEEPWIKI_OPEN_SYNC_EVALUATION_20260701.md` diff --git a/Dockerfile b/Dockerfile index 951f67cf6..93e7ae096 100644 --- a/Dockerfile +++ b/Dockerfile @@ -66,9 +66,12 @@ RUN if [ -n "${CUSTOM_CERT_DIR}" ]; then \ fi ENV PATH="/opt/venv/bin:$PATH" +ENV TIKTOKEN_CACHE_DIR=/opt/tiktoken_cache/ # Copy Python dependencies COPY --from=py_deps /api/.venv /opt/venv +RUN mkdir -p "$TIKTOKEN_CACHE_DIR" && \ + python -c "import tiktoken; tiktoken.get_encoding('cl100k_base')" COPY api/ ./api/ # Copy Node app diff --git a/Dockerfile-litellm b/Dockerfile-litellm new file mode 100644 index 000000000..6692b2d76 --- /dev/null +++ b/Dockerfile-litellm @@ -0,0 +1,125 @@ +# syntax=docker/dockerfile:1-labs + +FROM node:20-alpine AS node_base + +FROM node_base AS node_deps +WORKDIR /app +COPY package.json package-lock.json ./ +RUN npm ci --legacy-peer-deps + +FROM node_base AS node_builder +WORKDIR /app +COPY --from=node_deps /app/node_modules ./node_modules +COPY --exclude=./api . . +RUN NODE_ENV=production npm run build + +FROM python:3.11-slim AS py_deps +WORKDIR /api +COPY api/pyproject.toml . +COPY api/poetry.lock . +RUN python -m pip install poetry==2.0.1 --no-cache-dir && \ + poetry config virtualenvs.create true --local && \ + poetry config virtualenvs.in-project true --local && \ + poetry config virtualenvs.options.always-copy --local true && \ + POETRY_MAX_WORKERS=10 poetry install --no-interaction --no-ansi --only main && \ + poetry cache clear --all . + +FROM python:3.11-slim AS ollama_base +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl zstd && rm -rf /var/lib/apt/lists/* + +# Detect architecture and download appropriate Ollama version +# ARG TARGETARCH can be set at build time with --build-arg TARGETARCH=arm64 or TARGETARCH=amd64 +ARG TARGETARCH +RUN OLLAMA_ARCH="" && \ + if [ "$TARGETARCH" = "arm64" ]; then \ + echo "Building for ARM64 architecture." && \ + OLLAMA_ARCH="arm64"; \ + elif [ "$TARGETARCH" = "amd64" ]; then \ + echo "Building for AMD64 architecture." && \ + OLLAMA_ARCH="amd64"; \ + else \ + echo "Error: Unsupported architecture '$TARGETARCH'. Supported architectures are 'arm64' and 'amd64'." >&2 && \ + exit 1; \ + fi && \ + (set -o pipefail; \ + curl -fL "https://ollama.com/download/ollama-linux-${OLLAMA_ARCH}.tar.zst" \ + | zstd -d | tar -x -C /usr) + +RUN ollama serve > /dev/null 2>&1 & \ + sleep 20 && \ + ollama pull nomic-embed-text && \ + ollama pull qwen3:1.7b + +# Use Python 3.11 as final image +FROM python:3.11-slim + +# Set working directory +WORKDIR /app + +# Install Node.js and npm +RUN apt-get update && apt-get install -y \ + curl \ + gnupg \ + git \ + ca-certificates \ + && mkdir -p /etc/apt/keyrings \ + && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \ + && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_20.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list \ + && apt-get update \ + && apt-get install -y nodejs \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +ENV PATH="/opt/venv/bin:$PATH" +ENV TIKTOKEN_CACHE_DIR=/opt/tiktoken_cache/ + +# Copy Python dependencies +COPY --from=py_deps /api/.venv /opt/venv +RUN mkdir -p "$TIKTOKEN_CACHE_DIR" && \ + python -c "import tiktoken; tiktoken.get_encoding('cl100k_base')" +COPY api/ ./api/ + +# Copy Node app +COPY --from=node_builder /app/public ./public +COPY --from=node_builder /app/.next/standalone ./ +COPY --from=node_builder /app/.next/static ./.next/static +COPY --from=ollama_base /usr/bin/ollama /usr/local/bin/ +COPY --from=ollama_base /root/.ollama /root/.ollama + +# Expose the port the app runs on +EXPOSE ${PORT:-8001} 3000 + +# Create a script to run both backend and frontend +RUN echo '#!/bin/bash\n\ +# Start ollama serve in background\n\ +ollama serve > /dev/null 2>&1 &\n\ +\n\ +# Load environment variables from .env file if it exists\n\ +if [ -f .env ]; then\n\ + export $(grep -v "^#" .env | xargs -r)\n\ +fi\n\ +\n\ +# Check for required environment variables\n\ +if [ -z "$OPENAI_API_KEY" ] || [ -z "$GOOGLE_API_KEY" ]; then\n\ + echo "Warning: OPENAI_API_KEY and/or GOOGLE_API_KEY environment variables are not set."\n\ + echo "These are required for DeepWiki to function properly."\n\ + echo "You can provide them via a mounted .env file or as environment variables when running the container."\n\ +fi\n\ +\n\ +# Start the API server in the background with the configured port\n\ +python -m api.main --port ${PORT:-8001} &\n\ +PORT=3000 HOSTNAME=0.0.0.0 node server.js &\n\ +wait -n\n\ +exit $?' > /app/start.sh && chmod +x /app/start.sh + +# Set environment variables +ENV PORT=8001 +ENV NODE_ENV=production +ENV SERVER_BASE_URL=http://localhost:${PORT:-8001} + +# Create empty .env file (will be overridden if one exists at runtime) +RUN touch .env + +# Command to run the application +CMD ["/app/start.sh"] diff --git a/Dockerfile-ollama-local b/Dockerfile-ollama-local index fd2597fc5..98d047440 100644 --- a/Dockerfile-ollama-local +++ b/Dockerfile-ollama-local @@ -71,9 +71,12 @@ RUN apt-get update && apt-get install -y \ && rm -rf /var/lib/apt/lists/* ENV PATH="/opt/venv/bin:$PATH" +ENV TIKTOKEN_CACHE_DIR=/opt/tiktoken_cache/ # Copy Python dependencies COPY --from=py_deps /api/.venv /opt/venv +RUN mkdir -p "$TIKTOKEN_CACHE_DIR" && \ + python -c "import tiktoken; tiktoken.get_encoding('cl100k_base')" COPY api/ ./api/ # Copy Node app diff --git a/README.md b/README.md index ba8fd1a88..8abbe6740 100644 --- a/README.md +++ b/README.md @@ -1,15 +1,6 @@ +# DeepWiki-Open (Grok-Wiki) -### ⚠️ Announcement: Shifting focus to AsyncReview ---- - -**IMPORTANT UPDATE** DeepWiki-Open maintenance is ongoing, but primary active development is moving to **[AsyncReview](https://github.com/AsyncFuncAI/AsyncReview/)**. Thank you for the support on this project; please join me in the new repository for this year's primary effort. - ---- ---- - -# DeepWiki-Open - -![DeepWiki Banner](screenshots/Deepwiki.png) +grok-wiki **DeepWiki** is my own implementation attempt of DeepWiki, automatically creates beautiful, interactive wikis for any GitHub, GitLab, or BitBucket repository! Just enter a repo name, and DeepWiki will: @@ -25,631 +16,15 @@ [English](./README.md) | [简体中文](./README.zh.md) | [繁體中文](./README.zh-tw.md) | [日本語](./README.ja.md) | [Español](./README.es.md) | [한국어](./README.kr.md) | [Tiếng Việt](./README.vi.md) | [Português Brasileiro](./README.pt-br.md) | [Français](./README.fr.md) | [Русский](./README.ru.md) -## ✨ Features - -- **Instant Documentation**: Turn any GitHub, GitLab or BitBucket repo into a wiki in seconds -- **Private Repository Support**: Securely access private repositories with personal access tokens -- **Smart Analysis**: AI-powered understanding of code structure and relationships -- **Beautiful Diagrams**: Automatic Mermaid diagrams to visualize architecture and data flow -- **Easy Navigation**: Simple, intuitive interface to explore the wiki -- **Ask Feature**: Chat with your repository using RAG-powered AI to get accurate answers -- **DeepResearch**: Multi-turn research process that thoroughly investigates complex topics -- **Multiple Model Providers**: Support for Google Gemini, OpenAI, OpenRouter, and local Ollama models -- **Flexible Embeddings**: Choose between OpenAI, Google AI, or local Ollama embeddings for optimal performance - -## 🚀 Quick Start (Super Easy!) - -### Option 1: Using Docker - -```bash -# Clone the repository -git clone https://github.com/AsyncFuncAI/deepwiki-open.git -cd deepwiki-open - -# Create a .env file with your API keys -echo "GOOGLE_API_KEY=your_google_api_key" > .env -echo "OPENAI_API_KEY=your_openai_api_key" >> .env -# Optional: Use Google AI embeddings instead of OpenAI (recommended if using Google models) -echo "DEEPWIKI_EMBEDDER_TYPE=google" >> .env -# Optional: Add OpenRouter API key if you want to use OpenRouter models -echo "OPENROUTER_API_KEY=your_openrouter_api_key" >> .env -# Optional: Add Ollama host if not local. defaults to http://localhost:11434 -echo "OLLAMA_HOST=your_ollama_host" >> .env -# Optional: Add Azure API key, endpoint and version if you want to use azure openai models -echo "AZURE_OPENAI_API_KEY=your_azure_openai_api_key" >> .env -echo "AZURE_OPENAI_ENDPOINT=your_azure_openai_endpoint" >> .env -echo "AZURE_OPENAI_VERSION=your_azure_openai_version" >> .env -# Run with Docker Compose -docker-compose up -``` - -For detailed instructions on using DeepWiki with Ollama and Docker, see [Ollama Instructions](Ollama-instruction.md). - -> 💡 **Where to get these keys:** -> - Get a Google API key from [Google AI Studio](https://makersuite.google.com/app/apikey) -> - Get an OpenAI API key from [OpenAI Platform](https://platform.openai.com/api-keys) -> - Get Azure OpenAI credentials from [Azure Portal](https://portal.azure.com/) - create an Azure OpenAI resource and get the API key, endpoint, and API version - -### Option 2: Manual Setup (Recommended) - -#### Step 1: Set Up Your API Keys - -Create a `.env` file in the project root with these keys: - -``` -GOOGLE_API_KEY=your_google_api_key -OPENAI_API_KEY=your_openai_api_key -# Optional: Use Google AI embeddings (recommended if using Google models) -DEEPWIKI_EMBEDDER_TYPE=google -# Optional: Add this if you want to use OpenRouter models -OPENROUTER_API_KEY=your_openrouter_api_key -# Optional: Add this if you want to use Azure OpenAI models -AZURE_OPENAI_API_KEY=your_azure_openai_api_key -AZURE_OPENAI_ENDPOINT=your_azure_openai_endpoint -AZURE_OPENAI_VERSION=your_azure_openai_version -# Optional: Add Ollama host if not local. default: http://localhost:11434 -OLLAMA_HOST=your_ollama_host -``` - -#### Step 2: Start the Backend - -```bash -# Install Python dependencies -python -m pip install poetry==2.0.1 && poetry install -C api - -# Start the API server -python -m api.main -``` - -#### Step 3: Start the Frontend - -```bash -# Install JavaScript dependencies -npm install -# or -yarn install - -# Start the web app -npm run dev -# or -yarn dev -``` - -#### Step 4: Use DeepWiki! - -1. Open [http://localhost:3000](http://localhost:3000) in your browser -2. Enter a GitHub, GitLab, or Bitbucket repository (like `https://github.com/openai/codex`, `https://github.com/microsoft/autogen`, `https://gitlab.com/gitlab-org/gitlab`, or `https://bitbucket.org/redradish/atlassian_app_versions`) -3. For private repositories, click "+ Add access tokens" and enter your GitHub or GitLab personal access token -4. Click "Generate Wiki" and watch the magic happen! - -## 🔍 How It Works - -DeepWiki uses AI to: - -1. Clone and analyze the GitHub, GitLab, or Bitbucket repository (including private repos with token authentication) -2. Create embeddings of the code for smart retrieval -3. Generate documentation with context-aware AI (using Google Gemini, OpenAI, OpenRouter, Azure OpenAI, or local Ollama models) -4. Create visual diagrams to explain code relationships -5. Organize everything into a structured wiki -6. Enable intelligent Q&A with the repository through the Ask feature -7. Provide in-depth research capabilities with DeepResearch - -```mermaid -graph TD - A[User inputs GitHub/GitLab/Bitbucket repo] --> AA{Private repo?} - AA -->|Yes| AB[Add access token] - AA -->|No| B[Clone Repository] - AB --> B - B --> C[Analyze Code Structure] - C --> D[Create Code Embeddings] - - D --> M{Select Model Provider} - M -->|Google Gemini| E1[Generate with Gemini] - M -->|OpenAI| E2[Generate with OpenAI] - M -->|OpenRouter| E3[Generate with OpenRouter] - M -->|Local Ollama| E4[Generate with Ollama] - M -->|Azure| E5[Generate with Azure] - - E1 --> E[Generate Documentation] - E2 --> E - E3 --> E - E4 --> E - E5 --> E - - D --> F[Create Visual Diagrams] - E --> G[Organize as Wiki] - F --> G - G --> H[Interactive DeepWiki] - - classDef process stroke-width:2px; - classDef data stroke-width:2px; - classDef result stroke-width:2px; - classDef decision stroke-width:2px; - - class A,D data; - class AA,M decision; - class B,C,E,F,G,AB,E1,E2,E3,E4,E5 process; - class H result; -``` - -## 🛠️ Project Structure - -``` -deepwiki/ -├── api/ # Backend API server -│ ├── main.py # API entry point -│ ├── api.py # FastAPI implementation -│ ├── rag.py # Retrieval Augmented Generation -│ ├── data_pipeline.py # Data processing utilities -│ ├── pyproject.toml # Python dependencies (Poetry) -│ └── poetry.lock # Locked Python dependency versions -│ -├── src/ # Frontend Next.js app -│ ├── app/ # Next.js app directory -│ │ └── page.tsx # Main application page -│ └── components/ # React components -│ └── Mermaid.tsx # Mermaid diagram renderer -│ -├── public/ # Static assets -├── package.json # JavaScript dependencies -└── .env # Environment variables (create this) -``` - -## 🤖 Provider-Based Model Selection System - -DeepWiki now implements a flexible provider-based model selection system supporting multiple LLM providers: - -### Supported Providers and Models - -- **Google**: Default `gemini-2.5-flash`, also supports `gemini-2.5-flash-lite`, `gemini-2.5-pro`, etc. -- **OpenAI**: Default `gpt-5-nano`, also supports `gpt-5`, `4o`, etc. -- **OpenRouter**: Access to multiple models via a unified API, including Claude, Llama, Mistral, etc. -- **Azure OpenAI**: Default `gpt-4o`, also supports `o4-mini`, etc. -- **Ollama**: Support for locally running open-source models like `llama3` - -### Environment Variables - -Each provider requires its corresponding API key environment variables: - -``` -# API Keys -GOOGLE_API_KEY=your_google_api_key # Required for Google Gemini models -OPENAI_API_KEY=your_openai_api_key # Required for OpenAI models -OPENROUTER_API_KEY=your_openrouter_api_key # Required for OpenRouter models -AZURE_OPENAI_API_KEY=your_azure_openai_api_key #Required for Azure OpenAI models -AZURE_OPENAI_ENDPOINT=your_azure_openai_endpoint #Required for Azure OpenAI models -AZURE_OPENAI_VERSION=your_azure_openai_version #Required for Azure OpenAI models - -# OpenAI API Base URL Configuration -OPENAI_BASE_URL=https://custom-api-endpoint.com/v1 # Optional, for custom OpenAI API endpoints - -# Ollama host -OLLAMA_HOST=your_ollama_host # Optional, if Ollama is not local. default: http://localhost:11434 - -# Configuration Directory -DEEPWIKI_CONFIG_DIR=/path/to/custom/config/dir # Optional, for custom config file location -``` - -### Configuration Files - -DeepWiki uses JSON configuration files to manage various aspects of the system: - -1. **`generator.json`**: Configuration for text generation models - - Defines available model providers (Google, OpenAI, OpenRouter, Azure, Ollama) - - Specifies default and available models for each provider - - Contains model-specific parameters like temperature and top_p - -2. **`embedder.json`**: Configuration for embedding models and text processing - - Defines embedding models for vector storage - - Contains retriever configuration for RAG - - Specifies text splitter settings for document chunking - -3. **`repo.json`**: Configuration for repository handling - - Contains file filters to exclude certain files and directories - - Defines repository size limits and processing rules - -By default, these files are located in the `api/config/` directory. You can customize their location using the `DEEPWIKI_CONFIG_DIR` environment variable. - -### Custom Model Selection for Service Providers - -The custom model selection feature is specifically designed for service providers who need to: - -- You can offer multiple AI model choices to users within your organization -- You can quickly adapt to the rapidly evolving LLM landscape without code changes -- You can support specialized or fine-tuned models that aren't in the predefined list - -Service providers can implement their model offerings by selecting from the predefined options or entering custom model identifiers in the frontend interface. - -### Base URL Configuration for Enterprise Private Channels - -The OpenAI Client's base_url configuration is designed primarily for enterprise users with private API channels. This feature: - -- Enables connection to private or enterprise-specific API endpoints -- Allows organizations to use their own self-hosted or custom-deployed LLM services -- Supports integration with third-party OpenAI API-compatible services - -**Coming Soon**: In future updates, DeepWiki will support a mode where users need to provide their own API keys in requests. This will allow enterprise customers with private channels to use their existing API arrangements without sharing credentials with the DeepWiki deployment. - -## 🧩 Using OpenAI-Compatible Embedding Models (e.g., Alibaba Qwen) - -If you want to use embedding models compatible with the OpenAI API (such as Alibaba Qwen), follow these steps: - -1. Replace the contents of `api/config/embedder.json` with those from `api/config/embedder_openai_compatible.json`. -2. In your project root `.env` file, set the relevant environment variables, for example: - ``` - OPENAI_API_KEY=your_api_key - OPENAI_BASE_URL=your_openai_compatible_endpoint - ``` -3. The program will automatically substitute placeholders in embedder.json with the values from your environment variables. - -This allows you to seamlessly switch to any OpenAI-compatible embedding service without code changes. - -## 🧠 Using Google AI Embeddings - -DeepWiki now supports Google AI's latest embedding models as an alternative to OpenAI embeddings. This provides better integration when you're already using Google Gemini models for text generation. - -### Features - -- **Latest Model**: Uses Google's `text-embedding-004` model -- **Same API Key**: Uses your existing `GOOGLE_API_KEY` (no additional setup required) -- **Better Integration**: Optimized for use with Google Gemini text generation models -- **Task-Specific**: Supports semantic similarity, retrieval, and classification tasks -- **Batch Processing**: Efficient processing of multiple texts - -### How to Enable Google AI Embeddings - -**Option 1: Environment Variable (Recommended)** - -Set the embedder type in your `.env` file: - -```bash -# Your existing Google API key -GOOGLE_API_KEY=your_google_api_key - -# Enable Google AI embeddings -DEEPWIKI_EMBEDDER_TYPE=google -``` - -**Option 2: Docker Environment** - -```bash -docker run -p 8001:8001 -p 3000:3000 \ - -e GOOGLE_API_KEY=your_google_api_key \ - -e DEEPWIKI_EMBEDDER_TYPE=google \ - -v ~/.adalflow:/root/.adalflow \ - ghcr.io/asyncfuncai/deepwiki-open:latest -``` - -**Option 3: Docker Compose** - -Add to your `.env` file: - -```bash -GOOGLE_API_KEY=your_google_api_key -DEEPWIKI_EMBEDDER_TYPE=google -``` - -Then run: - -```bash -docker-compose up -``` - -### Available Embedder Types - -| Type | Description | API Key Required | Notes | -|------|-------------|------------------|-------| -| `openai` | OpenAI embeddings (default) | `OPENAI_API_KEY` | Uses `text-embedding-3-small` model | -| `google` | Google AI embeddings | `GOOGLE_API_KEY` | Uses `text-embedding-004` model | -| `ollama` | Local Ollama embeddings | None | Requires local Ollama installation | - -### Why Use Google AI Embeddings? - -- **Consistency**: If you're using Google Gemini for text generation, using Google embeddings provides better semantic consistency -- **Performance**: Google's latest embedding model offers excellent performance for retrieval tasks -- **Cost**: Competitive pricing compared to OpenAI -- **No Additional Setup**: Uses the same API key as your text generation models - -### Switching Between Embedders - -You can easily switch between different embedding providers: - -```bash -# Use OpenAI embeddings (default) -export DEEPWIKI_EMBEDDER_TYPE=openai - -# Use Google AI embeddings -export DEEPWIKI_EMBEDDER_TYPE=google - -# Use local Ollama embeddings -export DEEPWIKI_EMBEDDER_TYPE=ollama -``` - -**Note**: When switching embedders, you may need to regenerate your repository embeddings as different models produce different vector spaces. - -### Logging - -DeepWiki uses Python's built-in `logging` module for diagnostic output. You can configure the verbosity and log file destination via environment variables: - -| Variable | Description | Default | -|-----------------|--------------------------------------------------------------------|------------------------------| -| `LOG_LEVEL` | Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL). | INFO | -| `LOG_FILE_PATH` | Path to the log file. If set, logs will be written to this file. | `api/logs/application.log` | - -To enable debug logging and direct logs to a custom file: -```bash -export LOG_LEVEL=DEBUG -export LOG_FILE_PATH=./debug.log -python -m api.main -``` -Or with Docker Compose: -```bash -LOG_LEVEL=DEBUG LOG_FILE_PATH=./debug.log docker-compose up -``` - -When running with Docker Compose, the container's `api/logs` directory is bind-mounted to `./api/logs` on your host (see the `volumes` section in `docker-compose.yml`), ensuring log files persist across restarts. - -Alternatively, you can store these settings in your `.env` file: - -```bash -LOG_LEVEL=DEBUG -LOG_FILE_PATH=./debug.log -``` -Then simply run: - -```bash -docker-compose up -``` - -**Logging Path Security Considerations:** In production environments, ensure the `api/logs` directory and any custom log file path are secured with appropriate filesystem permissions and access controls. The application enforces that `LOG_FILE_PATH` resides within the project's `api/logs` directory to prevent path traversal or unauthorized writes. - -## 🛠️ Advanced Setup - -### Environment Variables - -| Variable | Description | Required | Note | -|----------------------|--------------------------------------------------------------|----------|----------------------------------------------------------------------------------------------------------| -| `GOOGLE_API_KEY` | Google Gemini API key for AI generation and embeddings | No | Required for Google Gemini models and Google AI embeddings -| `OPENAI_API_KEY` | OpenAI API key for embeddings and models | Conditional | Required if using OpenAI embeddings or models | -| `OPENROUTER_API_KEY` | OpenRouter API key for alternative models | No | Required only if you want to use OpenRouter models | -| `AWS_ACCESS_KEY_ID` | AWS access key ID for Bedrock | No | Required for Bedrock if not using instance/role-based credentials | -| `AWS_SECRET_ACCESS_KEY` | AWS secret access key for Bedrock | No | Required for Bedrock if not using instance/role-based credentials | -| `AWS_SESSION_TOKEN` | AWS session token for Bedrock (STS) | No | Required when using temporary credentials | -| `AWS_REGION` | AWS region for Bedrock (default: `us-east-1`) | No | Used by Bedrock client | -| `AWS_ROLE_ARN` | AWS role ARN to assume for Bedrock | No | If set, the Bedrock client will call STS AssumeRole | -| `AZURE_OPENAI_API_KEY` | Azure OpenAI API key | No | Required only if you want to use Azure OpenAI models | -| `AZURE_OPENAI_ENDPOINT` | Azure OpenAI endpoint | No | Required only if you want to use Azure OpenAI models | -| `AZURE_OPENAI_VERSION` | Azure OpenAI version | No | Required only if you want to use Azure OpenAI models | -| `OLLAMA_HOST` | Ollama Host (default: http://localhost:11434) | No | Required only if you want to use external Ollama server | -| `DEEPWIKI_EMBEDDER_TYPE` | Embedder type: `openai`, `google`, `ollama`, or `bedrock` (default: `openai`) | No | Controls which embedding provider to use | -| `PORT` | Port for the API server (default: 8001) | No | If you host API and frontend on the same machine, make sure change port of `SERVER_BASE_URL` accordingly | -| `SERVER_BASE_URL` | Base URL for the API server (default: http://localhost:8001) | No | -| `DEEPWIKI_AUTH_MODE` | Set to `true` or `1` to enable authorization mode. | No | Defaults to `false`. If enabled, `DEEPWIKI_AUTH_CODE` is required. | -| `DEEPWIKI_AUTH_CODE` | The secret code required for wiki generation when `DEEPWIKI_AUTH_MODE` is enabled. | No | Only used if `DEEPWIKI_AUTH_MODE` is `true` or `1`. | - -**API Key Requirements:** -- If using `DEEPWIKI_EMBEDDER_TYPE=openai` (default): `OPENAI_API_KEY` is required -- If using `DEEPWIKI_EMBEDDER_TYPE=google`: `GOOGLE_API_KEY` is required -- If using `DEEPWIKI_EMBEDDER_TYPE=ollama`: No API key required (local processing) -- If using `DEEPWIKI_EMBEDDER_TYPE=bedrock`: AWS credentials (or role-based credentials) are required - -Other API keys are only required when configuring and using models from the corresponding providers. - -## Authorization Mode - -DeepWiki can be configured to run in an authorization mode, where wiki generation requires a valid authorization code. This is useful if you want to control who can use the generation feature. -Restricts frontend initiation and protects cache deletion, but doesn't fully prevent backend generation if API endpoints are hit directly. - -To enable authorization mode, set the following environment variables: - -- `DEEPWIKI_AUTH_MODE`: Set this to `true` or `1`. When enabled, the frontend will display an input field for the authorization code. -- `DEEPWIKI_AUTH_CODE`: Set this to the desired secret code. Restricts frontend initiation and protects cache deletion, but doesn't fully prevent backend generation if API endpoints are hit directly. - -If `DEEPWIKI_AUTH_MODE` is not set or is set to `false` (or any other value than `true`/`1`), the authorization feature will be disabled, and no code will be required. - -### Docker Setup - -You can use Docker to run DeepWiki: - -#### Running the Container - -```bash -# Pull the image from GitHub Container Registry -docker pull ghcr.io/asyncfuncai/deepwiki-open:latest - -# Run the container with environment variables -docker run -p 8001:8001 -p 3000:3000 \ - -e GOOGLE_API_KEY=your_google_api_key \ - -e OPENAI_API_KEY=your_openai_api_key \ - -e OPENROUTER_API_KEY=your_openrouter_api_key \ - -e OLLAMA_HOST=your_ollama_host \ - -e AZURE_OPENAI_API_KEY=your_azure_openai_api_key \ - -e AZURE_OPENAI_ENDPOINT=your_azure_openai_endpoint \ - -e AZURE_OPENAI_VERSION=your_azure_openai_version \ - - -v ~/.adalflow:/root/.adalflow \ - ghcr.io/asyncfuncai/deepwiki-open:latest -``` - -This command also mounts `~/.adalflow` on your host to `/root/.adalflow` in the container. This path is used to store: -- Cloned repositories (`~/.adalflow/repos/`) -- Their embeddings and indexes (`~/.adalflow/databases/`) -- Cached generated wiki content (`~/.adalflow/wikicache/`) - -This ensures that your data persists even if the container is stopped or removed. - -Or use the provided `docker-compose.yml` file: - -```bash -# Edit the .env file with your API keys first -docker-compose up -``` - -(The `docker-compose.yml` file is pre-configured to mount `~/.adalflow` for data persistence, similar to the `docker run` command above.) - -#### Using a .env file with Docker - -You can also mount a .env file to the container: - -```bash -# Create a .env file with your API keys -echo "GOOGLE_API_KEY=your_google_api_key" > .env -echo "OPENAI_API_KEY=your_openai_api_key" >> .env -echo "OPENROUTER_API_KEY=your_openrouter_api_key" >> .env -echo "AZURE_OPENAI_API_KEY=your_azure_openai_api_key" >> .env -echo "AZURE_OPENAI_ENDPOINT=your_azure_openai_endpoint" >> .env -echo "AZURE_OPENAI_VERSION=your_azure_openai_version" >>.env -echo "OLLAMA_HOST=your_ollama_host" >> .env - -# Run the container with the .env file mounted -docker run -p 8001:8001 -p 3000:3000 \ - -v $(pwd)/.env:/app/.env \ - -v ~/.adalflow:/root/.adalflow \ - ghcr.io/asyncfuncai/deepwiki-open:latest -``` - -This command also mounts `~/.adalflow` on your host to `/root/.adalflow` in the container. This path is used to store: -- Cloned repositories (`~/.adalflow/repos/`) -- Their embeddings and indexes (`~/.adalflow/databases/`) -- Cached generated wiki content (`~/.adalflow/wikicache/`) - -This ensures that your data persists even if the container is stopped or removed. - -#### Building the Docker image locally - -If you want to build the Docker image locally: - -```bash -# Clone the repository -git clone https://github.com/AsyncFuncAI/deepwiki-open.git -cd deepwiki-open - -# Build the Docker image -docker build -t deepwiki-open . - -# Run the container -docker run -p 8001:8001 -p 3000:3000 \ - -e GOOGLE_API_KEY=your_google_api_key \ - -e OPENAI_API_KEY=your_openai_api_key \ - -e OPENROUTER_API_KEY=your_openrouter_api_key \ - -e AZURE_OPENAI_API_KEY=your_azure_openai_api_key \ - -e AZURE_OPENAI_ENDPOINT=your_azure_openai_endpoint \ - -e AZURE_OPENAI_VERSION=your_azure_openai_version \ - -e OLLAMA_HOST=your_ollama_host \ - deepwiki-open -``` - -#### Using Self-Signed Certificates in Docker - -If you're in an environment that uses self-signed certificates, you can include them in the Docker build: - -1. Create a directory for your certificates (default is `certs` in your project root) -2. Copy your `.crt` or `.pem` certificate files into this directory -3. Build the Docker image: - -```bash -# Build with default certificates directory (certs) -docker build . - -# Or build with a custom certificates directory -docker build --build-arg CUSTOM_CERT_DIR=my-custom-certs . -``` - -### API Server Details - -The API server provides: -- Repository cloning and indexing -- RAG (Retrieval Augmented Generation) -- Streaming chat completions - -For more details, see the [API README](./api/README.md). - -## 🔌 OpenRouter Integration - -DeepWiki now supports [OpenRouter](https://openrouter.ai/) as a model provider, giving you access to hundreds of AI models through a single API: - -- **Multiple Model Options**: Access models from OpenAI, Anthropic, Google, Meta, Mistral, and more -- **Simple Configuration**: Just add your OpenRouter API key and select the model you want to use -- **Cost Efficiency**: Choose models that fit your budget and performance needs -- **Easy Switching**: Toggle between different models without changing your code - -### How to Use OpenRouter with DeepWiki - -1. **Get an API Key**: Sign up at [OpenRouter](https://openrouter.ai/) and get your API key -2. **Add to Environment**: Add `OPENROUTER_API_KEY=your_key` to your `.env` file -3. **Enable in UI**: Check the "Use OpenRouter API" option on the homepage -4. **Select Model**: Choose from popular models like GPT-4o, Claude 3.5 Sonnet, Gemini 2.0, and more - -OpenRouter is particularly useful if you want to: -- Try different models without signing up for multiple services -- Access models that might be restricted in your region -- Compare performance across different model providers -- Optimize for cost vs. performance based on your needs - -## 🤖 Ask & DeepResearch Features - -### Ask Feature - -The Ask feature allows you to chat with your repository using Retrieval Augmented Generation (RAG): - -- **Context-Aware Responses**: Get accurate answers based on the actual code in your repository -- **RAG-Powered**: The system retrieves relevant code snippets to provide grounded responses -- **Real-Time Streaming**: See responses as they're generated for a more interactive experience -- **Conversation History**: The system maintains context between questions for more coherent interactions - -### DeepResearch Feature - -DeepResearch takes repository analysis to the next level with a multi-turn research process: - -- **In-Depth Investigation**: Thoroughly explores complex topics through multiple research iterations -- **Structured Process**: Follows a clear research plan with updates and a comprehensive conclusion -- **Automatic Continuation**: The AI automatically continues research until reaching a conclusion (up to 5 iterations) -- **Research Stages**: - 1. **Research Plan**: Outlines the approach and initial findings - 2. **Research Updates**: Builds on previous iterations with new insights - 3. **Final Conclusion**: Provides a comprehensive answer based on all iterations - -To use DeepResearch, simply toggle the "Deep Research" switch in the Ask interface before submitting your question. - -## Screenshots - -![DeepWiki Main Interface](screenshots/Interface.png) -*The main interface of DeepWiki* - -![Private Repository Support](screenshots/privaterepo.png) -*Access private repositories with personal access tokens* - -![DeepResearch Feature](screenshots/DeepResearch.png) -*DeepResearch conducts multi-turn investigations for complex topics* - -### Demo Video - -[![DeepWiki Demo Video](https://img.youtube.com/vi/zGANs8US8B4/0.jpg)](https://youtu.be/zGANs8US8B4) +## Deepwiki-Open 2.0 (Grok Wiki is now live) -*Watch DeepWiki in action!* +- **Download at** https://grok-wiki.com -## ❓ Troubleshooting -### API Key Issues -- **"Missing environment variables"**: Make sure your `.env` file is in the project root and contains the required API keys -- **"API key not valid"**: Check that you've copied the full key correctly with no extra spaces -- **"OpenRouter API error"**: Verify your OpenRouter API key is valid and has sufficient credits -- **"Azure OpenAI API error"**: Verify your Azure OpenAI credentials (API key, endpoint, and version) are correct and the service is properly deployed -### Connection Problems -- **"Cannot connect to API server"**: Make sure the API server is running on port 8001 -- **"CORS error"**: The API is configured to allow all origins, but if you're having issues, try running both frontend and backend on the same machine +https://github.com/user-attachments/assets/48d1e60a-eb91-4c05-a5a8-3624ffb79fb1 -### Generation Issues -- **"Error generating wiki"**: For very large repositories, try a smaller one first -- **"Invalid repository format"**: Make sure you're using a valid GitHub, GitLab or Bitbucket URL format -- **"Could not fetch repository structure"**: For private repositories, ensure you've entered a valid personal access token with appropriate permissions -- **"Diagram rendering error"**: The app will automatically try to fix broken diagrams -### Common Solutions -1. **Restart both servers**: Sometimes a simple restart fixes most issues -2. **Check console logs**: Open browser developer tools to see any JavaScript errors -3. **Check API logs**: Look at the terminal where the API is running for Python errors ## 🤝 Contributing diff --git a/api/README.md b/api/README.md index 65e82a2dc..76aa389e4 100644 --- a/api/README.md +++ b/api/README.md @@ -30,6 +30,7 @@ OPENAI_API_KEY=your_openai_api_key # Required for embeddings and OpenAI m # Optional API Keys OPENROUTER_API_KEY=your_openrouter_api_key # Required only if using OpenRouter models +MINIMAX_API_KEY=your_minimax_key # Required only if using MiniMax models # AWS Bedrock Configuration AWS_ACCESS_KEY_ID=your_aws_access_key_id # Required for AWS Bedrock models @@ -40,6 +41,9 @@ AWS_ROLE_ARN=your_aws_role_arn # Optional, for role-based authent # OpenAI API Configuration OPENAI_BASE_URL=https://custom-api-endpoint.com/v1 # Optional, for custom OpenAI API endpoints +# MiniMax API Configuration +MINIMAX_BASE_URL=https://api.minimax.io/v1 # Optional, defaults to the official MiniMax OpenAI-compatible endpoint + # Ollama host OLLAMA_HOST=https://your_ollama_host" # Optional: Add Ollama host if not local. default: http://localhost:11434 @@ -52,6 +56,7 @@ If you're not using Ollama mode, you need to configure an OpenAI API key for emb > 💡 **Where to get these keys:** > - Get a Google API key from [Google AI Studio](https://makersuite.google.com/app/apikey) > - Get an OpenAI API key from [OpenAI Platform](https://platform.openai.com/api-keys) +> - Get a MiniMax API key or subscription key from the MiniMax platform console > - Get an OpenRouter API key from [OpenRouter](https://openrouter.ai/keys) > - Get AWS credentials from [AWS IAM Console](https://console.aws.amazon.com/iam/) @@ -62,6 +67,7 @@ DeepWiki supports multiple LLM providers. The environment variables above are re - **Google Gemini**: Requires `GOOGLE_API_KEY` - **OpenAI**: Requires `OPENAI_API_KEY` +- **MiniMax**: Requires `MINIMAX_API_KEY`; defaults to `MiniMax-M3` through the OpenAI-compatible MiniMax endpoint - **OpenRouter**: Requires `OPENROUTER_API_KEY` - **AWS Bedrock**: Requires `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` - **Ollama**: No API key required (runs locally) @@ -83,7 +89,7 @@ DeepWiki now uses JSON configuration files to manage various system components i 1. **`generator.json`**: Configuration for text generation models - Located in `api/config/` by default - - Defines available model providers (Google, OpenAI, OpenRouter, AWS Bedrock, Ollama) + - Defines available model providers (Google, OpenAI, MiniMax, OpenRouter, AWS Bedrock, Ollama) - Specifies default and available models for each provider - Contains model-specific parameters like temperature and top_p diff --git a/api/api.py b/api/api.py index d40e73f96..0c26f4208 100644 --- a/api/api.py +++ b/api/api.py @@ -398,7 +398,7 @@ def generate_json_export(repo_url: str, pages: List[WikiPage]) -> str: app.add_api_route("/chat/completions/stream", chat_completions_stream, methods=["POST"]) # Add the WebSocket endpoint -app.add_websocket_route("/ws/chat", handle_websocket_chat) +app.add_api_websocket_route("/ws/chat", handle_websocket_chat) # --- Wiki Cache Helper Functions --- diff --git a/api/config.py b/api/config.py index 49dfcf7b0..14b7e3a32 100644 --- a/api/config.py +++ b/api/config.py @@ -8,6 +8,8 @@ logger = logging.getLogger(__name__) from api.openai_client import OpenAIClient +from api.litellm_client import LiteLLMClient +from api.minimax_client import MiniMaxClient from api.openrouter_client import OpenRouterClient from api.bedrock_client import BedrockClient from api.google_embedder_client import GoogleEmbedderClient @@ -17,6 +19,8 @@ # Get API keys from environment variables OPENAI_API_KEY = os.environ.get('OPENAI_API_KEY') +LITELLM_API_KEY = os.environ.get('LITELLM_API_KEY') +MINIMAX_API_KEY = os.environ.get('MINIMAX_API_KEY') GOOGLE_API_KEY = os.environ.get('GOOGLE_API_KEY') OPENROUTER_API_KEY = os.environ.get('OPENROUTER_API_KEY') AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID') @@ -28,6 +32,10 @@ # Set keys in environment (in case they're needed elsewhere in the code) if OPENAI_API_KEY: os.environ["OPENAI_API_KEY"] = OPENAI_API_KEY +if LITELLM_API_KEY: + os.environ["LITELLM_API_KEY"] = LITELLM_API_KEY +if MINIMAX_API_KEY: + os.environ["MINIMAX_API_KEY"] = MINIMAX_API_KEY if GOOGLE_API_KEY: os.environ["GOOGLE_API_KEY"] = GOOGLE_API_KEY if OPENROUTER_API_KEY: @@ -59,6 +67,8 @@ "GoogleGenAIClient": GoogleGenAIClient, "GoogleEmbedderClient": GoogleEmbedderClient, "OpenAIClient": OpenAIClient, + "LiteLLMClient" : LiteLLMClient, + "MiniMaxClient": MiniMaxClient, "OpenRouterClient": OpenRouterClient, "OllamaClient": OllamaClient, "BedrockClient": BedrockClient, @@ -131,10 +141,12 @@ def load_generator_config(): if provider_config.get("client_class") in CLIENT_CLASSES: provider_config["model_client"] = CLIENT_CLASSES[provider_config["client_class"]] # Fall back to default mapping based on provider_id - elif provider_id in ["google", "openai", "openrouter", "ollama", "bedrock", "azure", "dashscope"]: + elif provider_id in ["google", "openai", "openrouter", "ollama", "bedrock", "azure", "dashscope", "litellm", "minimax"]: default_map = { "google": GoogleGenAIClient, "openai": OpenAIClient, + "litellm": LiteLLMClient, + "minimax": MiniMaxClient, "openrouter": OpenRouterClient, "ollama": OllamaClient, "bedrock": BedrockClient, @@ -361,7 +373,7 @@ def get_model_config(provider="google", model=None): Get configuration for the specified provider and model Parameters: - provider (str): Model provider ('google', 'openai', 'openrouter', 'ollama', 'bedrock') + provider (str): Model provider ('google', 'openai', 'openrouter', 'ollama', 'bedrock', 'minimax') model (str): Model name, or None to use default model Returns: diff --git a/api/config/generator.json b/api/config/generator.json index f88179098..32d836a4a 100644 --- a/api/config/generator.json +++ b/api/config/generator.json @@ -73,6 +73,31 @@ } } }, + "minimax": { + "client_class": "MiniMaxClient", + "default_model": "MiniMax-M3", + "supportsCustomModel": true, + "models": { + "MiniMax-M3": { + "temperature": 0.7, + "top_p": 0.95, + "max_completion_tokens": 4096, + "thinking": { + "type": "disabled" + } + }, + "MiniMax-M2.7": { + "temperature": 0.7, + "top_p": 0.9, + "max_completion_tokens": 4096 + }, + "MiniMax-M2.5": { + "temperature": 0.7, + "top_p": 0.9, + "max_completion_tokens": 4096 + } + } + }, "openrouter": { "default_model": "openai/gpt-5-nano", "supportsCustomModel": true, @@ -196,4 +221,3 @@ } } } - diff --git a/api/config/repo.json b/api/config/repo.json index 52c31503b..b1afea014 100644 --- a/api/config/repo.json +++ b/api/config/repo.json @@ -11,7 +11,8 @@ "./.git/", "./.svn/", "./.hg/", - "./.bzr/" + "./.bzr/", + "./vendor/" ], "excluded_files": [ "yarn.lock", diff --git a/api/data_pipeline.py b/api/data_pipeline.py index 5e1f5fa47..f98068651 100644 --- a/api/data_pipeline.py +++ b/api/data_pipeline.py @@ -116,8 +116,16 @@ def download_repo(repo_url: str, local_path: str, repo_type: str = None, access_ # Format: https://oauth2:{token}@gitlab.com/owner/repo.git clone_url = urlunparse((parsed.scheme, f"oauth2:{encoded_token}@{parsed.netloc}", parsed.path, '', '', '')) elif repo_type == "bitbucket": - # Format: https://x-token-auth:{token}@bitbucket.org/owner/repo.git - clone_url = urlunparse((parsed.scheme, f"x-token-auth:{encoded_token}@{parsed.netloc}", parsed.path, '', '', '')) + # Bitbucket has two token formats with different auth schemes: + # - HTTP access tokens (prefix "ATCTT") use x-bitbucket-api-token-auth + # - App passwords (deprecated, EOL June 2026) use x-token-auth + # Detect by token prefix so existing app password users keep working. + if access_token.startswith("ATCTT"): + auth_scheme = "x-bitbucket-api-token-auth" + else: + auth_scheme = "x-token-auth" + # Format: https://{auth_scheme}:{token}@bitbucket.org/owner/repo.git + clone_url = urlunparse((parsed.scheme, f"{auth_scheme}:{encoded_token}@{parsed.netloc}", parsed.path, '', '', '')) logger.info("Using access token for authentication") diff --git a/api/litellm_client.py b/api/litellm_client.py new file mode 100644 index 000000000..439adced6 --- /dev/null +++ b/api/litellm_client.py @@ -0,0 +1,67 @@ +import os +from typing import Optional, Callable +from openai import AsyncOpenAI, OpenAI + +from api.openai_client import OpenAIClient + + +class LiteLLMClient(OpenAIClient): + """ + LiteLLM OpenAI-compatible client. + + LiteLLM exposes an OpenAI-compatible API surface, so we can + reuse almost all OpenAIClient behavior while overriding only + the client initialization. + + Expected environment variables: + + LITELLM_BASE_URL=http://litellm:4000 + LITELLM_API_KEY=sk-1234 + + Example model names: + openai/gpt-4o + anthropic/claude-3-5-sonnet + gemini/gemini-2.5-pro + ollama/llama3 + """ + + def __init__( + self, + api_key: Optional[str] = None, + chat_completion_parser: Optional[Callable] = None, + input_type: str = "text", + base_url: Optional[str] = None, + env_base_url_name: str = "LITELLM_BASE_URL", + env_api_key_name: str = "LITELLM_API_KEY", + ): + resolved_base_url = base_url or os.getenv(env_base_url_name, "http://localhost:4000") + if not resolved_base_url.endswith("/v1"): + resolved_base_url = f"{resolved_base_url.rstrip('/')}/v1" + super().__init__( + api_key=api_key, + chat_completion_parser=chat_completion_parser, + input_type=input_type, + base_url=resolved_base_url, + env_base_url_name=env_base_url_name, + env_api_key_name=env_api_key_name, + ) + + def init_sync_client(self): + """ + Initialize synchronous LiteLLM OpenAI-compatible client. + """ + api_key = self._api_key or os.getenv(self._env_api_key_name, "dummy") + return OpenAI( + api_key=api_key, + base_url=self.base_url, + ) + + def init_async_client(self): + """ + Initialize asynchronous LiteLLM OpenAI-compatible client. + """ + api_key = self._api_key or os.getenv(self._env_api_key_name, "dummy") + return AsyncOpenAI( + api_key=api_key, + base_url=self.base_url, + ) diff --git a/api/minimax_client.py b/api/minimax_client.py new file mode 100644 index 000000000..b42484f8b --- /dev/null +++ b/api/minimax_client.py @@ -0,0 +1,106 @@ +"""MiniMax OpenAI-compatible ModelClient integration.""" + +import os +from typing import Any, Callable, Dict, Literal, Optional + +from api.openai_client import OpenAIClient + + +class MiniMaxClient(OpenAIClient): + """ + MiniMax OpenAI-compatible client. + + MiniMax serves MiniMax-M3 through an OpenAI-compatible Chat Completions + endpoint. Keep this as a direct provider so MiniMax token-plan usage does + not depend on a LiteLLM proxy or shared OPENAI_API_KEY. + """ + + MINIMAX_EXTRA_BODY_KEYS = {"thinking", "reasoning_split", "service_tier"} + + def __init__( + self, + api_key: Optional[str] = None, + chat_completion_parser: Optional[Callable] = None, + input_type: str = "text", + base_url: Optional[str] = None, + env_base_url_name: str = "MINIMAX_BASE_URL", + env_api_key_name: str = "MINIMAX_API_KEY", + ): + resolved_base_url = base_url or os.getenv( + env_base_url_name, "https://api.minimax.io/v1" + ) + super().__init__( + api_key=api_key, + chat_completion_parser=chat_completion_parser, + input_type=input_type, + base_url=resolved_base_url.rstrip("/"), + env_base_url_name=env_base_url_name, + env_api_key_name=env_api_key_name, + ) + + def convert_inputs_to_api_kwargs( + self, + input: Optional[Any] = None, + model_kwargs: Dict = {}, + model_type=None, + ) -> Dict: + api_kwargs = super().convert_inputs_to_api_kwargs( + input=input, + model_kwargs=model_kwargs, + model_type=model_type, + ) + + extra_body = dict(api_kwargs.pop("extra_body", {}) or {}) + for key in self.MINIMAX_EXTRA_BODY_KEYS: + if key in api_kwargs: + extra_body[key] = api_kwargs.pop(key) + + if extra_body: + api_kwargs["extra_body"] = extra_body + + return api_kwargs + + +class MiniMaxStreamTextExtractor: + """Extract display text from MiniMax streaming chunks. + + MiniMax's OpenAI-compatible examples show cumulative ``delta.content`` + chunks. Some OpenAI-compatible clients emit incremental deltas. This helper + accepts both forms and returns only the newly visible text. + """ + + def __init__(self): + self._buffer = "" + self._mode: Optional[Literal["cumulative", "incremental"]] = None + + def extract(self, text: Optional[str]) -> str: + if not text: + return "" + + if not self._buffer: + self._buffer = text + return text + + if self._mode == "cumulative": + if text.startswith(self._buffer): + new_text = text[len(self._buffer):] + self._buffer = text + return new_text + + self._mode = "incremental" + self._buffer += text + return text + + if self._mode == "incremental": + self._buffer += text + return text + + if text.startswith(self._buffer) and len(text) > len(self._buffer): + self._mode = "cumulative" + new_text = text[len(self._buffer):] + self._buffer = text + return new_text + + self._mode = "incremental" + self._buffer += text + return text diff --git a/api/simple_chat.py b/api/simple_chat.py index 41a184ed8..d9f0c3667 100644 --- a/api/simple_chat.py +++ b/api/simple_chat.py @@ -11,8 +11,9 @@ from fastapi.responses import StreamingResponse from pydantic import BaseModel, Field -from api.config import get_model_config, configs, OPENROUTER_API_KEY, OPENAI_API_KEY, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY +from api.config import get_model_config, configs, OPENROUTER_API_KEY, OPENAI_API_KEY, MINIMAX_API_KEY, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY from api.data_pipeline import count_tokens, get_file_content +from api.minimax_client import MiniMaxClient, MiniMaxStreamTextExtractor from api.openai_client import OpenAIClient from api.openrouter_client import OpenRouterClient from api.bedrock_client import BedrockClient @@ -64,7 +65,7 @@ class ChatCompletionRequest(BaseModel): type: Optional[str] = Field("github", description="Type of repository (e.g., 'github', 'gitlab', 'bitbucket')") # model parameters - provider: str = Field("google", description="Model provider (google, openai, openrouter, ollama, bedrock, azure, dashscope)") + provider: str = Field("google", description="Model provider (google, openai, minimax, openrouter, ollama, bedrock, azure, dashscope)") model: Optional[str] = Field(None, description="Model name for the specified provider") language: Optional[str] = Field("en", description="Language for content generation (e.g., 'en', 'ja', 'zh', 'es', 'kr', 'vi')") @@ -390,6 +391,27 @@ async def chat_completions_stream(request: ChatCompletionRequest): if "top_p" in model_config: model_kwargs["top_p"] = model_config["top_p"] + api_kwargs = model.convert_inputs_to_api_kwargs( + input=prompt, + model_kwargs=model_kwargs, + model_type=ModelType.LLM + ) + elif request.provider == "minimax": + logger.info(f"Using MiniMax protocol with model: {model_config['model']}") + + if not MINIMAX_API_KEY: + logger.warning("MINIMAX_API_KEY not configured, but continuing with request") + + model = MiniMaxClient() + model_kwargs = { + "model": model_config["model"], + "stream": True, + "temperature": model_config["temperature"], + } + for key in ["top_p", "max_completion_tokens", "thinking", "reasoning_split", "service_tier"]: + if key in model_config: + model_kwargs[key] = model_config[key] + api_kwargs = model.convert_inputs_to_api_kwargs( input=prompt, model_kwargs=model_kwargs, @@ -500,6 +522,23 @@ async def response_stream(): except Exception as e_openai: logger.error(f"Error with Openai API: {str(e_openai)}") yield f"\nError with Openai API: {str(e_openai)}\n\nPlease check that you have set the OPENAI_API_KEY environment variable with a valid API key." + elif request.provider == "minimax": + try: + logger.info("Making MiniMax API call") + response = await model.acall(api_kwargs=api_kwargs, model_type=ModelType.LLM) + stream_text = MiniMaxStreamTextExtractor() + async for chunk in response: + choices = getattr(chunk, "choices", []) + if len(choices) > 0: + delta = getattr(choices[0], "delta", None) + if delta is not None: + text = getattr(delta, "content", None) + new_text = stream_text.extract(text) + if new_text: + yield new_text + except Exception as e_minimax: + logger.error(f"Error with MiniMax API: {str(e_minimax)}") + yield f"\nError with MiniMax API: {str(e_minimax)}\n\nPlease check that you have set the MINIMAX_API_KEY environment variable with a valid API key." elif request.provider == "bedrock": try: # Get the response and handle it properly using the previously created api_kwargs @@ -635,6 +674,30 @@ async def response_stream(): except Exception as e_fallback: logger.error(f"Error with Openai API fallback: {str(e_fallback)}") yield f"\nError with Openai API fallback: {str(e_fallback)}\n\nPlease check that you have set the OPENAI_API_KEY environment variable with a valid API key." + elif request.provider == "minimax": + try: + fallback_api_kwargs = model.convert_inputs_to_api_kwargs( + input=simplified_prompt, + model_kwargs=model_kwargs, + model_type=ModelType.LLM + ) + + logger.info("Making fallback MiniMax API call") + fallback_response = await model.acall(api_kwargs=fallback_api_kwargs, model_type=ModelType.LLM) + + stream_text = MiniMaxStreamTextExtractor() + async for chunk in fallback_response: + choices = getattr(chunk, "choices", []) + if len(choices) > 0: + delta = getattr(choices[0], "delta", None) + if delta is not None: + text = getattr(delta, "content", None) + new_text = stream_text.extract(text) + if new_text: + yield new_text + except Exception as e_fallback: + logger.error(f"Error with MiniMax API fallback: {str(e_fallback)}") + yield f"\nError with MiniMax API fallback: {str(e_fallback)}\n\nPlease check that you have set the MINIMAX_API_KEY environment variable with a valid API key." elif request.provider == "bedrock": try: # Create new api_kwargs with the simplified prompt diff --git a/api/websocket_wiki.py b/api/websocket_wiki.py index d1a6b1bd5..cef07799c 100644 --- a/api/websocket_wiki.py +++ b/api/websocket_wiki.py @@ -14,12 +14,16 @@ configs, OPENROUTER_API_KEY, OPENAI_API_KEY, + LITELLM_API_KEY, + MINIMAX_API_KEY, AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, ) from api.data_pipeline import count_tokens, get_file_content from api.bedrock_client import BedrockClient +from api.minimax_client import MiniMaxClient, MiniMaxStreamTextExtractor from api.openai_client import OpenAIClient +from api.litellm_client import LiteLLMClient from api.openrouter_client import OpenRouterClient from api.azureai_client import AzureAIClient from api.dashscope_client import DashscopeClient @@ -50,7 +54,7 @@ class ChatCompletionRequest(BaseModel): # model parameters provider: str = Field( "google", - description="Model provider (google, openai, openrouter, ollama, bedrock, azure, dashscope)", + description="Model provider (google, openai, minimax, openrouter, ollama, bedrock, azure, dashscope)", ) model: Optional[str] = Field(None, description="Model name for the specified provider") @@ -500,6 +504,51 @@ async def handle_websocket_chat(websocket: WebSocket): if "top_p" in model_config: model_kwargs["top_p"] = model_config["top_p"] + api_kwargs = model.convert_inputs_to_api_kwargs( + input=prompt, + model_kwargs=model_kwargs, + model_type=ModelType.LLM + ) + elif request.provider == "litellm": + logger.info(f"Using Openai protocol with model on LiteLLM: {request.model}") + + # Check if an API key is set for Litellm + if not LITELLM_API_KEY: + logger.warning("LITELLM_API_KEY not configured, but continuing with request") + # We'll let the OpenAIClient handle this and return an error message + + # Initialize LiteLLM client + model = LiteLLMClient() + model_kwargs = { + "model": request.model, + "stream": True, + "temperature": model_config["temperature"] + } + # Only add top_p if it exists in the model config + if "top_p" in model_config: + model_kwargs["top_p"] = model_config["top_p"] + + api_kwargs = model.convert_inputs_to_api_kwargs( + input=prompt, + model_kwargs=model_kwargs, + model_type=ModelType.LLM + ) + elif request.provider == "minimax": + logger.info(f"Using MiniMax protocol with model: {model_config['model']}") + + if not MINIMAX_API_KEY: + logger.warning("MINIMAX_API_KEY not configured, but continuing with request") + + model = MiniMaxClient() + model_kwargs = { + "model": model_config["model"], + "stream": True, + "temperature": model_config["temperature"], + } + for key in ["top_p", "max_completion_tokens", "thinking", "reasoning_split", "service_tier"]: + if key in model_config: + model_kwargs[key] = model_config[key] + api_kwargs = model.convert_inputs_to_api_kwargs( input=prompt, model_kwargs=model_kwargs, @@ -640,6 +689,48 @@ async def handle_websocket_chat(websocket: WebSocket): await websocket.send_text(error_msg) # Close the WebSocket connection after sending the error message await websocket.close() + elif request.provider == "litellm": + try: + # Get the response and handle it properly using the previously created api_kwargs + logger.info("Making LiteLLM API call") + response = await model.acall(api_kwargs=api_kwargs, model_type=ModelType.LLM) + # Handle streaming response from LiteLLM + async for chunk in response: + choices = getattr(chunk, "choices", []) + if len(choices) > 0: + delta = getattr(choices[0], "delta", None) + if delta is not None: + text = getattr(delta, "content", None) + if text is not None: + await websocket.send_text(text) + # Explicitly close the WebSocket connection after the response is complete + await websocket.close() + except Exception as e_litellm: + logger.error(f"Error with LiteLLM API: {str(e_litellm)}") + error_msg = f"\nError with LiteLLM API: {str(e_litellm)}\n\nPlease check that you have set the LITELLM_API_KEY environment variable with a valid API key." + await websocket.send_text(error_msg) + # Close the WebSocket connection after sending the error message + await websocket.close() + elif request.provider == "minimax": + try: + logger.info("Making MiniMax API call") + response = await model.acall(api_kwargs=api_kwargs, model_type=ModelType.LLM) + stream_text = MiniMaxStreamTextExtractor() + async for chunk in response: + choices = getattr(chunk, "choices", []) + if len(choices) > 0: + delta = getattr(choices[0], "delta", None) + if delta is not None: + text = getattr(delta, "content", None) + new_text = stream_text.extract(text) + if new_text: + await websocket.send_text(new_text) + await websocket.close() + except Exception as e_minimax: + logger.error(f"Error with MiniMax API: {str(e_minimax)}") + error_msg = f"\nError with MiniMax API: {str(e_minimax)}\n\nPlease check that you have set the MINIMAX_API_KEY environment variable with a valid API key." + await websocket.send_text(error_msg) + await websocket.close() elif request.provider == "bedrock": try: logger.info("Making AWS Bedrock API call") @@ -793,6 +884,57 @@ async def handle_websocket_chat(websocket: WebSocket): logger.error(f"Error with Openai API fallback: {str(e_fallback)}") error_msg = f"\nError with Openai API fallback: {str(e_fallback)}\n\nPlease check that you have set the OPENAI_API_KEY environment variable with a valid API key." await websocket.send_text(error_msg) + elif request.provider == "litellm": + try: + # Create new api_kwargs with the simplified prompt + fallback_api_kwargs = model.convert_inputs_to_api_kwargs( + input=simplified_prompt, + model_kwargs=model_kwargs, + model_type=ModelType.LLM + ) + + # Get the response using the simplified prompt + logger.info("Making fallback LiteLLM API call") + fallback_response = await model.acall(api_kwargs=fallback_api_kwargs, model_type=ModelType.LLM) + + # Handle streaming fallback_response from LiteLLM + async for chunk in fallback_response: + choices = getattr(chunk, "choices", []) + if len(choices) > 0: + delta = getattr(choices[0], "delta", None) + if delta is not None: + text = getattr(delta, "content", None) + if text is not None: + await websocket.send_text(text) + except Exception as e_fallback: + logger.error(f"Error with LiteLLM API fallback: {str(e_fallback)}") + error_msg = f"\nError with LiteLLM API fallback: {str(e_fallback)}\n\nPlease check that you have set the LITELLM_API_KEY environment variable with a valid API key." + await websocket.send_text(error_msg) + elif request.provider == "minimax": + try: + fallback_api_kwargs = model.convert_inputs_to_api_kwargs( + input=simplified_prompt, + model_kwargs=model_kwargs, + model_type=ModelType.LLM + ) + + logger.info("Making fallback MiniMax API call") + fallback_response = await model.acall(api_kwargs=fallback_api_kwargs, model_type=ModelType.LLM) + + stream_text = MiniMaxStreamTextExtractor() + async for chunk in fallback_response: + choices = getattr(chunk, "choices", []) + if len(choices) > 0: + delta = getattr(choices[0], "delta", None) + if delta is not None: + text = getattr(delta, "content", None) + new_text = stream_text.extract(text) + if new_text: + await websocket.send_text(new_text) + except Exception as e_fallback: + logger.error(f"Error with MiniMax API fallback: {str(e_fallback)}") + error_msg = f"\nError with MiniMax API fallback: {str(e_fallback)}\n\nPlease check that you have set the MINIMAX_API_KEY environment variable with a valid API key." + await websocket.send_text(error_msg) elif request.provider == "bedrock": try: fallback_api_kwargs = model.convert_inputs_to_api_kwargs( diff --git a/archive/verification/static_scan_delete_apis_20260701.tsv b/archive/verification/static_scan_delete_apis_20260701.tsv new file mode 100644 index 000000000..cd0b28219 --- /dev/null +++ b/archive/verification/static_scan_delete_apis_20260701.tsv @@ -0,0 +1 @@ +FilePath LineNumber Pattern MatchedCode Verdict diff --git a/docker-compose-litellm.env b/docker-compose-litellm.env new file mode 100644 index 000000000..0a246ec4e --- /dev/null +++ b/docker-compose-litellm.env @@ -0,0 +1,7 @@ +# Same Compose network http://litellm:4000 +# LiteLLM on host OS http://host.docker.internal:4000 +# LiteLLM on another server http://server-ip:4000 +LITELLM_BASE_URL=http://litellm:4000 +# Use env. variable LITELLM_API_KEY +# But if it doesn't exist, default to `sk-1234` +LITELLM_API_KEY=${LITELLM_API_KEY:-sk-1234} \ No newline at end of file diff --git a/docker-compose-litellm.yml b/docker-compose-litellm.yml new file mode 100644 index 000000000..6f90878a2 --- /dev/null +++ b/docker-compose-litellm.yml @@ -0,0 +1,70 @@ +services: + db: + image: postgres:16 + environment: + POSTGRES_DB: litellm + POSTGRES_USER: litellm + # Use env. variable LITELLM_DB_PASSWORD + # otherwise default to litellm_password + POSTGRES_PASSWORD: ${LITELLM_DB_PASSWORD:-litellm_password} + ports: + - "5432:5432" + volumes: + - deepwiki_litellm_db:/var/lib/postgresql/data + + litellm: + image: ghcr.io/berriai/litellm:v1.83.10-stable + ports: + - "4000:4000" + volumes: + - ./litellm-config.yml:/app/config.yaml + environment: + # Use env. variable LITELLM_MASTER_KEY + # otherwise default to sk-1234 + - LITELLM_MASTER_KEY=${LITELLM_MASTER_KEY:-sk-1234} + # Using from litellm-config.yml + #environment: + # DATABASE_URL: postgres://litellm:litellm_password@db:5432/litellm + command: [ "--config", "/app/config.yaml", "--port", "4000" ] + extra_hosts: + - "host.docker.internal:host-gateway" + depends_on: + - db + + deepwiki: + image: deepwiki-litellm # Using image name to show it depends on LiteLLM + build: + context: . + dockerfile: Dockerfile-litellm # ← Using Dockerfile that depends on LiteLLM + ports: + - "${PORT:-8001}:${PORT:-8001}" # API port + - "3000:3000" # Next.js port + env_file: + - docker-compose-litellm.env + environment: + - PORT=${PORT:-8001} + - NODE_ENV=production + - SERVER_BASE_URL=http://localhost:${PORT:-8001} + - LOG_LEVEL=${LOG_LEVEL:-INFO} + - LOG_FILE_PATH=${LOG_FILE_PATH:-api/logs/application.log} + # For LiteLLM + - LITELLM_BASE_URL=${LITELLM_BASE_URL} + - LITELLM_API_KEY=${LITELLM_API_KEY} + volumes: + - ~/.adalflow:/root/.adalflow # Persist repository and embedding data + - ./api/logs:/app/api/logs # Persist log files across container restarts + # Resource limits for docker-compose up (not Swarm mode) + mem_limit: 6g + mem_reservation: 2g + # Health check configuration + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:${PORT:-8001}/health"] + interval: 60s + timeout: 10s + retries: 3 + start_period: 30s + depends_on: + - litellm + +volumes: + deepwiki_litellm_db: \ No newline at end of file diff --git a/docs/DEEPWIKI_OPEN_SYNC_EVALUATION_20260701.md b/docs/DEEPWIKI_OPEN_SYNC_EVALUATION_20260701.md new file mode 100644 index 000000000..5f36d3e4b --- /dev/null +++ b/docs/DEEPWIKI_OPEN_SYNC_EVALUATION_20260701.md @@ -0,0 +1,138 @@ +# DeepWiki Open Sync Evaluation - 2026-07-01 + +## Verdict + +CONSIDER the upstream sync branch for local review only. The branch cleanly +merges AsyncFuncAI `upstream/main` through `16f35a0` and preserves the local MCP +wrapper seams, but upstream still has security-relevant open issues and npm +audit findings that should block production or broad-network exposure. + +2026-07-02 update: the LiteLLM lane should not become the default local +provider path. This branch now adds a direct `minimax` provider for +`MiniMax-M3` using MiniMax's OpenAI-compatible Chat Completions endpoint and +separate `MINIMAX_API_KEY` / `MINIMAX_BASE_URL` environment variables. Keep the +upstream LiteLLM compose/client artifacts experimental or local-lab only unless +a separate LiteLLM hardening branch is approved. + +2026-07-02 live smoke update: MiniMax M3 was tested with `MINIMAX_API_KEY` +loaded only into a one-shot Python process from the approved central interim +env source. The smoke returned `minimax_smoke=pass`, `model=MiniMax-M3`, +`content_length=2`, and `total_tokens=170`. No credential value was printed, +committed, hcom-sent, or persisted in this repository. + +Do not replace this fork wholesale with `AIDotNet/OpenDeepWiki`. AIDotNet is a +larger .NET/Next.js knowledge-base platform with admin, database, users, API +keys, workers, and built-in MCP endpoints. It is worth isolated evaluation and +selective extraction, not a drop-in replacement for this local FastAPI/Next.js +DeepWiki lab. + +## Evidence Ledger + +| Claim | Source | Status | +|---|---|---| +| Local fork was clean on `main` and aligned with `origin/main` before sync. | `git status --short --branch` in `/home/riche/MCPs/deepwiki-open` | verified | +| Hosted DeepWiki MCP was initially absent from live Codex config, then restored by the Codex config owner. Current config shows `deepwiki` enabled as streamable HTTP. | `codex mcp list`; `codex mcp get deepwiki` | verified | +| The current running Codex session still does not expose native `mcp__deepwiki__` tools, but the hosted MCP endpoint responds directly. | direct streamable HTTP `initialize`, `tools/list`, `read_wiki_structure`, and `ask_question` probes to `https://mcp.deepwiki.com/mcp` | verified | +| Merge-base with AsyncFuncAI upstream is `05591ee`; local is 17 commits ahead and upstream is 8 commits ahead after unshallowing. | `git merge-base HEAD upstream/main`; `git rev-list --count` | verified | +| Upstream sync merged with no conflicts in worktree `/home/riche/MCPs/deepwiki-open/WORKTREES/upstream-20260701`. | `git merge upstream/main --no-ff --no-commit`; `git diff --name-only --diff-filter=U` | verified | +| Upstream adds LiteLLM support and Docker tiktoken cache preload. | `Dockerfile`, `Dockerfile-litellm`, `api/litellm_client.py`, `docker-compose-litellm.yml` | verified | +| Local bome patch adds direct MiniMax M3 provider wiring without reusing `OPENAI_API_KEY` or LiteLLM. | `api/minimax_client.py`, `api/config/generator.json`, `api/simple_chat.py`, `api/websocket_wiki.py` | verified by focused tests | +| Local Dockerfile startup-secret hardening is preserved while upstream tiktoken cache preload is added. | `Dockerfile:68-105` | verified | +| Local `mcp_wrapper.py` cache invalidation now archives old `.pkl` files instead of deleting them. | `mcp_wrapper.py:_invalidate_cache` | verified | +| Static no-delete gate passes after the wrapper change. | `archive/verification/static_scan_delete_apis_20260701.tsv` | verified | +| Frontend build passes after `npm ci --ignore-scripts`. | `npm run build` in sync worktree | verified | +| npm audit reports 23 vulnerabilities: 17 moderate, 5 high, 1 critical. | `npm audit --audit-level=moderate --json` | verified | +| AsyncFuncAI upstream is MIT, active, no latest release, 17,113 stars, 1,922 forks, pushed 2026-06-03. | `gh repo view AsyncFuncAI/deepwiki-open` | verified | +| AsyncFuncAI has open security-relevant reports, including unauthenticated file-read/git-clone SSRF issue #536 and dependency issues #537/#539/#500. | `gh issue list -R AsyncFuncAI/deepwiki-open` | verified public reports | +| DeepWiki MCP describes AsyncFuncAI as a Python FastAPI + Next.js RAG system with FAISS/local storage, multiple provider integrations, permissive CORS, token-based private repo clone support, and exposed 8001/3000 Docker ports. | hosted DeepWiki MCP `ask_question(repoName="AsyncFuncAI/deepwiki-open", ...)` | verified | +| AIDotNet/OpenDeepWiki is MIT, active, latest release `v2.0.3` on 2026-05-30, 3,376 stars, 432 forks, pushed 2026-06-13. | `gh repo view AIDotNet/OpenDeepWiki`; `gh release list` | verified | +| AIDotNet uses .NET 10, Next.js 16/React 19, SQLite/PostgreSQL, LibGit2Sharp, background workers, admin, chat, and MCP endpoints. | `AIDotNet/OpenDeepWiki` README and source via `gh api` | verified | +| DeepWiki MCP describes AIDotNet as a .NET 10/Next.js 16 platform with Docker Compose install, AI provider binding, Git/ZIP/local ingestion, MCP endpoints, permissive CORS, local path root allowlisting, and plaintext `AuthPassword` repository credential storage. | hosted DeepWiki MCP `ask_question(repoName="AIDotNet/OpenDeepWiki", ...)` | verified | + +## Local Fork Findings + +- The local fork's value is concentrated in `mcp_wrapper.py`, + `docker-compose.override.yml`, `requirements-mcp.txt`, `.env.sample`, + `CLAUDE.md`, `FYI.md`, and `docs/`. +- The wrapper has useful safety seams: explicit host path allowlist, stream + aggregation limits, stale-cache warning, ephemeral-document exclusions, and + cost telemetry. +- Security surfaces remain important: auth is default-off upstream, CORS is + permissive, the local API can touch local source trees, and browser token URL + handling remains a credential-leak risk. +- The sync branch changes cache refresh behavior from delete to archive, which + keeps `force_refresh=True` semantics while satisfying archive-not-delete + policy. + +## Upstream AsyncFuncAI Findings + +- The upstream main branch is active but not release-managed. +- Hosted DeepWiki MCP corroborates the source-read architecture: FastAPI + backend, Next.js frontend, RAG ingestion, FAISS/local storage, and configurable + Google/OpenAI/OpenRouter/AWS Bedrock/Ollama/Azure provider coupling. +- The 8 upstream commits since `05591ee` are low-conflict and useful for local + review: LiteLLM support, Docker tiktoken cache preload, vendor exclusion, a + README "2.0" section, and Bitbucket token-scheme handling. Local follow-up + now routes the preferred extra provider through direct MiniMax M3 instead of + promoting LiteLLM as the happy path. +- Public open issues and audit output make upstream unsafe as-is for any + network-exposed deployment. +- The best path is selective sync plus local hardening, not blind upstream + parity. + +## AIDotNet/OpenDeepWiki Comparison + +- AIDotNet is stronger for a persistent self-hosted portal: admin console, + users, roles, departments, API keys, OAuth, GitHub App imports, background + processing, scheduled updates, public docs, chat APIs, and MCP endpoints. +- Hosted DeepWiki MCP corroborates that AIDotNet has built-in MCP endpoints, + AI provider/model binding, Git/ZIP/local ingestion, and local path root + allowlisting. +- AIDotNet is heavier operationally: .NET 10, Next.js 16, database migrations, + Docker registry images, background workers, and broader credential/config + surfaces. +- Security posture is mixed: BCrypt and hashed API keys are good, but default + JWT/admin bootstrap, permissive CORS, root compose runtime, plaintext repo + credential fields, disabled Git TLS verification, and Graphify subprocess + execution need hardening before use on sensitive machines. + +## Verification + +| Command | Result | +|---|---| +| `python3 /home/riche/.codex/scripts/static_check_no_delete_api.py --target-dir ...` | PASS after archiving cache invalidation | +| `python3 -m py_compile mcp_wrapper.py api/data_pipeline.py api/websocket_wiki.py api/litellm_client.py` | PASS | +| `/home/riche/MCPs/deepwiki-open/.venv/bin/python -c '... import mcp_wrapper ...'` | PASS | +| `/home/riche/MCPs/deepwiki-open/.venv/bin/python -m pytest tests/unit/test_minimax_provider.py -q` | PASS; 3 tests, 1 existing Google SDK deprecation warning | +| `/home/riche/MCPs/deepwiki-open/.venv/bin/python -m py_compile api/minimax_client.py api/config.py api/simple_chat.py api/websocket_wiki.py api/api.py` | PASS | +| `/home/riche/MCPs/deepwiki-open/.venv/bin/python -m json.tool api/config/generator.json` | PASS | +| MiniMax M3 live smoke with central env injection | PASS; returned `MiniMax-M3`, content length 2, total tokens 170, no secret output | +| FastAPI app config smoke for `/models/config` | PASS after switching WebSocket registration to `add_api_websocket_route`; response includes provider `minimax` and model `MiniMax-M3` | +| FastAPI route-level smoke for `/chat/completions/stream` with provider `minimax` | PASS with in-process retrieval patching and live MiniMax generation; returned `minimax route smoke ok` through the app route with no secret output | +| MiniMax stream parsing review | PASS after adding `MiniMaxStreamTextExtractor`; handles MiniMax cumulative `delta.content` chunks as shown in official MiniMax OpenAI SDK docs after a strictly growing cumulative chunk proves that stream mode, while preserving ambiguous repeated and suffix-overlap incremental chunks | +| `npm ci --ignore-scripts` | PASS; reported 23 vulnerabilities | +| `npm run build` | PASS with existing lint warnings | +| `npm audit --audit-level=moderate --json` | FAIL by audit policy: 23 vulnerabilities | + +## Recommended Next Actions + +1. Review and merge the sync branch only for local, non-public use after the + MiniMax provider patch is reviewed. +2. Configure MiniMax with `MINIMAX_API_KEY` and optional `MINIMAX_BASE_URL`; + select provider `minimax` and model `MiniMax-M3` for the underused MiniMax + token plan. +3. Run a true RAG-backed `/chat/completions/stream` smoke once the + retriever/embedder prerequisites are intentionally selected. Current blocker: + default OpenAI/Google embedding keys are not approved for this lane and local + Ollama does not have the configured `nomic-embed-text` embedder. +4. Keep upstream LiteLLM artifacts experimental/local-lab only until a separate + branch fixes config reachability, compose env handling, and exposure/default + credential risk. +5. Prioritize a hardening branch before exposing the service: localhost-only + binding by default, auth required on retrieval/cache mutation, narrow CORS, + local path allowlist enforcement, SSRF/private-IP blocking, and token removal + from browser URLs. +6. Treat npm/Next dependency remediation as a separate security task. The build + is green, but the audit result is not. +7. Evaluate AIDotNet in an isolated container or VM if the desired target is an + admin/team portal; extract patterns rather than migrating immediately. diff --git a/litellm-config.yml b/litellm-config.yml new file mode 100644 index 000000000..004bccdea --- /dev/null +++ b/litellm-config.yml @@ -0,0 +1,26 @@ +model_list: + # ----------------------- + # Chat / Generation models + # ----------------------- + - model_name: qwen3:1.7b #Can be named anything - qwen3-1.7b + litellm_params: + model: ollama/qwen3:1.7b + api_base: http://host.docker.internal:11434 + api_key: "not-needed" + + # ----------------------- + # Embedding models + # ----------------------- + - model_name: nomic-embed-text + litellm_params: + model: ollama/nomic-embed-text + api_base: http://host.docker.internal:11434 + api_key: "not-needed" + +# This overrides the one in the docker-compose file +general_settings: + database_url: postgres://litellm:litellm_password@db:5432/litellm + +litellm_settings: + set_verbose: true + drop_params: true diff --git a/mcp_wrapper.py b/mcp_wrapper.py index 5b64d7e3b..c25dedb86 100644 --- a/mcp_wrapper.py +++ b/mcp_wrapper.py @@ -259,15 +259,19 @@ def _check_cache_staleness(repo_host_path: str | None, db_path: Path) -> dict | def _invalidate_cache(repo_name: str) -> bool: - """Delete the cached embedding DB for force-refresh. Returns True if a file was removed.""" + """Archive the cached embedding DB for force-refresh. Returns True if moved aside.""" db_path = ADALFLOW_DB_DIR / f"{repo_name}.pkl" if db_path.exists(): try: - db_path.unlink() - logger.info("Cache invalidated: %s", db_path) + archive_dir = ADALFLOW_DB_DIR / "archive" + archive_dir.mkdir(parents=True, exist_ok=True) + timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ") + archive_path = archive_dir / f"{repo_name}.{timestamp}.pkl" + db_path.replace(archive_path) + logger.info("Cache archived for refresh: %s -> %s", db_path, archive_path) return True except OSError as exc: - logger.warning("Failed to invalidate cache %s: %s", db_path, exc) + logger.warning("Failed to archive cache %s: %s", db_path, exc) return False diff --git a/tests/unit/test_minimax_provider.py b/tests/unit/test_minimax_provider.py new file mode 100644 index 000000000..559eda351 --- /dev/null +++ b/tests/unit/test_minimax_provider.py @@ -0,0 +1,101 @@ +from adalflow.core.types import ModelType + +from api.config import get_model_config +from api.minimax_client import MiniMaxClient, MiniMaxStreamTextExtractor + + +def test_minimax_provider_config_defaults_to_m3(): + config = get_model_config("minimax") + + assert config["model_client"] is MiniMaxClient + assert config["model_kwargs"]["model"] == "MiniMax-M3" + assert config["model_kwargs"]["max_completion_tokens"] == 4096 + assert config["model_kwargs"]["thinking"] == {"type": "disabled"} + + +def test_minimax_client_uses_minimax_env_and_default_endpoint(monkeypatch): + monkeypatch.delenv("MINIMAX_BASE_URL", raising=False) + + client = MiniMaxClient(api_key="test-token") + + assert client.base_url == "https://api.minimax.io/v1" + assert client._env_api_key_name == "MINIMAX_API_KEY" + assert client._env_base_url_name == "MINIMAX_BASE_URL" + + +def test_minimax_client_uses_minimax_key_without_openai_key(monkeypatch): + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setenv("MINIMAX_API_KEY", "test-minimax-token") + + client = MiniMaxClient() + + assert client.sync_client is not None + assert client._env_api_key_name == "MINIMAX_API_KEY" + + +def test_minimax_client_moves_minimax_params_to_extra_body(): + client = MiniMaxClient(api_key="test-token") + + api_kwargs = client.convert_inputs_to_api_kwargs( + input="Explain this repository.", + model_type=ModelType.LLM, + model_kwargs={ + "model": "MiniMax-M3", + "stream": True, + "temperature": 0.7, + "thinking": {"type": "disabled"}, + "reasoning_split": True, + }, + ) + + assert api_kwargs["model"] == "MiniMax-M3" + assert api_kwargs["stream"] is True + assert api_kwargs["temperature"] == 0.7 + assert api_kwargs["extra_body"] == { + "thinking": {"type": "disabled"}, + "reasoning_split": True, + } + assert "thinking" not in api_kwargs + assert "reasoning_split" not in api_kwargs + assert api_kwargs["messages"] == [ + {"role": "user", "content": "Explain this repository."} + ] + + +def test_minimax_stream_text_extractor_handles_cumulative_chunks(): + extractor = MiniMaxStreamTextExtractor() + + assert extractor.extract("mini") == "mini" + assert extractor.extract("minimax") == "max" + assert extractor.extract("minimax route") == " route" + assert extractor.extract("minimax route") == "" + + +def test_minimax_stream_text_extractor_handles_incremental_chunks(): + extractor = MiniMaxStreamTextExtractor() + + assert extractor.extract("mini") == "mini" + assert extractor.extract("max") == "max" + assert extractor.extract(" route") == " route" + + +def test_minimax_stream_text_extractor_preserves_repeated_incremental_chunks(): + extractor = MiniMaxStreamTextExtractor() + + assert extractor.extract("ha") == "ha" + assert extractor.extract("ha") == "ha" + + +def test_minimax_stream_text_extractor_preserves_overlap_incremental_chunks(): + extractor = MiniMaxStreamTextExtractor() + + assert extractor.extract("abc") == "abc" + assert extractor.extract("cde") == "cde" + + +def test_minimax_stream_text_extractor_handles_empty_chunks(): + extractor = MiniMaxStreamTextExtractor() + + assert extractor.extract("minimax") == "minimax" + assert extractor.extract("") == "" + assert extractor.extract(None) == ""