From 573fc0937c11bba4f1590ad1bb97d8b39382c7ac Mon Sep 17 00:00:00 2001 From: Brian Gebel Date: Sat, 14 Mar 2026 19:50:24 -0700 Subject: [PATCH 1/7] feat(plugins): complete CGC plugin extension system (001-cgc-plugin-extension) Implements the full plugin extension system allowing third-party packages to contribute CLI commands and MCP tools to CGC via Python entry points. Core infrastructure: - PluginRegistry: discovers cgc_cli_plugins/cgc_mcp_plugins entry-point groups, validates PLUGIN_METADATA, enforces version constraints, isolates broken plugins - CLI integration: plugin commands attached to Typer app at import time; `cgc plugin list` - MCP server integration: plugin tools merged into tools dict; handlers routed at call time Plugins implemented: - cgc-plugin-stub: minimal reference fixture for testing and authoring examples - cgc-plugin-otel: gRPC OTLP receiver, Neo4j writer (Service/Trace/Span nodes, CORRELATES_TO links to static Method nodes), MCP query tools - cgc-plugin-xdebug: DBGp TCP listener, call-stack parser, dedup writer (StackFrame nodes, CALLED_BY chains, RESOLVES_TO Method links), dev-only - cgc-plugin-memory: MCP knowledge store (Memory/Observation nodes, DESCRIBES edges to Class/Method, FULLTEXT search, undocumented code queries) CI/CD and deployment: - GitHub Actions: plugin-publish.yml (matrix build/smoke-test/push via services.json), test-plugins.yml (PR plugin unit+integration tests) - Docker: docker-compose.plugin-stack.yml (self-contained full stack with Neo4j healthcheck + init.cypher), plugin/dev overlays, all plugin Dockerfiles - Kubernetes: otel-processor and memory plugin Deployment+Service manifests - Fixed docker-compose.template.yml: neo4j healthcheck was missing (broke all overlay depends_on: service_healthy conditions), init.cypher now mounted Tests (74 passing, 17 skipped pending plugin installs): - Unit: PluginRegistry, OTEL span processor, Xdebug DBGp parser - Integration: OTEL neo4j_writer, memory MCP handlers, plugin load isolation - E2E: full plugin lifecycle, broken plugin isolation, MCP server routing Documentation: - docs/plugins/authoring-guide.md: step-by-step plugin authoring with examples - docs/plugins/cross-layer-queries.md: 5 canonical cross-layer Cypher queries (SC-005) - docs/plugins/manual-testing.md: Docker and Python testing paths, per-plugin verification sequences, troubleshooting table - docs/plugins/examples/send_test_span.py: synthetic OTLP span sender for OTEL testing - .env.example: documented all plugin environment variables - CLAUDE.md: updated with plugin directories, entry-point groups, test commands Co-Authored-By: Claude Sonnet 4.6 --- .env.example | 22 + .github/services.json | 34 + .github/workflows/plugin-publish.yml | 127 +++ .github/workflows/test-plugins.yml | 84 ++ CLAUDE.md | 77 ++ cgc-extended-spec.md | 770 ++++++++++++++++++ config/neo4j/init.cypher | 38 + config/otel-collector/config.yaml | 33 + docker-compose.dev.yml | 38 + docker-compose.plugin-stack.yml | 159 ++++ docker-compose.plugins.yml | 86 ++ docker-compose.template.yml | 14 +- docs/plugins/authoring-guide.md | 226 +++++ docs/plugins/cross-layer-queries.md | 173 ++++ docs/plugins/examples/send_test_span.py | 72 ++ docs/plugins/manual-testing.md | 268 ++++++ k8s/cgc-plugin-memory/deployment.yaml | 67 ++ k8s/cgc-plugin-memory/service.yaml | 16 + k8s/cgc-plugin-otel/deployment.yaml | 66 ++ k8s/cgc-plugin-otel/service.yaml | 20 + plugins/cgc-plugin-memory/Dockerfile | 20 + plugins/cgc-plugin-memory/pyproject.toml | 35 + .../src/cgc_plugin_memory/__init__.py | 12 + .../src/cgc_plugin_memory/cli.py | 86 ++ .../src/cgc_plugin_memory/mcp_tools.py | 213 +++++ plugins/cgc-plugin-otel/Dockerfile | 22 + plugins/cgc-plugin-otel/pyproject.toml | 40 + .../src/cgc_plugin_otel/__init__.py | 11 + .../src/cgc_plugin_otel/cli.py | 83 ++ .../src/cgc_plugin_otel/mcp_tools.py | 128 +++ .../src/cgc_plugin_otel/neo4j_writer.py | 197 +++++ .../src/cgc_plugin_otel/receiver.py | 166 ++++ .../src/cgc_plugin_otel/span_processor.py | 103 +++ plugins/cgc-plugin-stub/pyproject.toml | 31 + .../src/cgc_plugin_stub/__init__.py | 8 + .../src/cgc_plugin_stub/cli.py | 15 + .../src/cgc_plugin_stub/mcp_tools.py | 37 + plugins/cgc-plugin-xdebug/Dockerfile | 21 + plugins/cgc-plugin-xdebug/pyproject.toml | 35 + .../src/cgc_plugin_xdebug/__init__.py | 16 + .../src/cgc_plugin_xdebug/cli.py | 90 ++ .../src/cgc_plugin_xdebug/dbgp_server.py | 223 +++++ .../src/cgc_plugin_xdebug/mcp_tools.py | 101 +++ .../src/cgc_plugin_xdebug/neo4j_writer.py | 142 ++++ pyproject.toml | 16 + .../checklists/requirements.md | 48 ++ .../contracts/cicd-pipeline.md | 149 ++++ .../contracts/plugin-interface.md | 241 ++++++ specs/001-cgc-plugin-extension/data-model.md | 320 ++++++++ specs/001-cgc-plugin-extension/plan.md | 176 ++++ specs/001-cgc-plugin-extension/quickstart.md | 211 +++++ specs/001-cgc-plugin-extension/research.md | 266 ++++++ specs/001-cgc-plugin-extension/spec.md | 362 ++++++++ specs/001-cgc-plugin-extension/tasks.md | 318 ++++++++ src/codegraphcontext/cli/main.py | 53 ++ src/codegraphcontext/plugin_registry.py | 283 +++++++ src/codegraphcontext/server.py | 38 +- tests/e2e/plugin/__init__.py | 0 tests/e2e/plugin/test_plugin_lifecycle.py | 445 ++++++++++ .../plugin/test_memory_integration.py | 148 ++++ .../plugin/test_otel_integration.py | 187 +++++ tests/integration/plugin/test_plugin_load.py | 210 +++++ tests/unit/plugin/test_otel_processor.py | 185 +++++ tests/unit/plugin/test_plugin_registry.py | 275 +++++++ tests/unit/plugin/test_xdebug_parser.py | 139 ++++ 65 files changed, 8288 insertions(+), 7 deletions(-) create mode 100644 .github/services.json create mode 100644 .github/workflows/plugin-publish.yml create mode 100644 .github/workflows/test-plugins.yml create mode 100644 CLAUDE.md create mode 100644 cgc-extended-spec.md create mode 100644 config/neo4j/init.cypher create mode 100644 config/otel-collector/config.yaml create mode 100644 docker-compose.dev.yml create mode 100644 docker-compose.plugin-stack.yml create mode 100644 docker-compose.plugins.yml create mode 100644 docs/plugins/authoring-guide.md create mode 100644 docs/plugins/cross-layer-queries.md create mode 100644 docs/plugins/examples/send_test_span.py create mode 100644 docs/plugins/manual-testing.md create mode 100644 k8s/cgc-plugin-memory/deployment.yaml create mode 100644 k8s/cgc-plugin-memory/service.yaml create mode 100644 k8s/cgc-plugin-otel/deployment.yaml create mode 100644 k8s/cgc-plugin-otel/service.yaml create mode 100644 plugins/cgc-plugin-memory/Dockerfile create mode 100644 plugins/cgc-plugin-memory/pyproject.toml create mode 100644 plugins/cgc-plugin-memory/src/cgc_plugin_memory/__init__.py create mode 100644 plugins/cgc-plugin-memory/src/cgc_plugin_memory/cli.py create mode 100644 plugins/cgc-plugin-memory/src/cgc_plugin_memory/mcp_tools.py create mode 100644 plugins/cgc-plugin-otel/Dockerfile create mode 100644 plugins/cgc-plugin-otel/pyproject.toml create mode 100644 plugins/cgc-plugin-otel/src/cgc_plugin_otel/__init__.py create mode 100644 plugins/cgc-plugin-otel/src/cgc_plugin_otel/cli.py create mode 100644 plugins/cgc-plugin-otel/src/cgc_plugin_otel/mcp_tools.py create mode 100644 plugins/cgc-plugin-otel/src/cgc_plugin_otel/neo4j_writer.py create mode 100644 plugins/cgc-plugin-otel/src/cgc_plugin_otel/receiver.py create mode 100644 plugins/cgc-plugin-otel/src/cgc_plugin_otel/span_processor.py create mode 100644 plugins/cgc-plugin-stub/pyproject.toml create mode 100644 plugins/cgc-plugin-stub/src/cgc_plugin_stub/__init__.py create mode 100644 plugins/cgc-plugin-stub/src/cgc_plugin_stub/cli.py create mode 100644 plugins/cgc-plugin-stub/src/cgc_plugin_stub/mcp_tools.py create mode 100644 plugins/cgc-plugin-xdebug/Dockerfile create mode 100644 plugins/cgc-plugin-xdebug/pyproject.toml create mode 100644 plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/__init__.py create mode 100644 plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/cli.py create mode 100644 plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/dbgp_server.py create mode 100644 plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/mcp_tools.py create mode 100644 plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/neo4j_writer.py create mode 100644 specs/001-cgc-plugin-extension/checklists/requirements.md create mode 100644 specs/001-cgc-plugin-extension/contracts/cicd-pipeline.md create mode 100644 specs/001-cgc-plugin-extension/contracts/plugin-interface.md create mode 100644 specs/001-cgc-plugin-extension/data-model.md create mode 100644 specs/001-cgc-plugin-extension/plan.md create mode 100644 specs/001-cgc-plugin-extension/quickstart.md create mode 100644 specs/001-cgc-plugin-extension/research.md create mode 100644 specs/001-cgc-plugin-extension/spec.md create mode 100644 specs/001-cgc-plugin-extension/tasks.md create mode 100644 src/codegraphcontext/plugin_registry.py create mode 100644 tests/e2e/plugin/__init__.py create mode 100644 tests/e2e/plugin/test_plugin_lifecycle.py create mode 100644 tests/integration/plugin/test_memory_integration.py create mode 100644 tests/integration/plugin/test_otel_integration.py create mode 100644 tests/integration/plugin/test_plugin_load.py create mode 100644 tests/unit/plugin/test_otel_processor.py create mode 100644 tests/unit/plugin/test_plugin_registry.py create mode 100644 tests/unit/plugin/test_xdebug_parser.py diff --git a/.env.example b/.env.example index 5eacc242..0b7315c6 100644 --- a/.env.example +++ b/.env.example @@ -34,3 +34,25 @@ PYTHONDONTWRITEBYTECODE=1 # Optional: Database selection # DATABASE_TYPE=falkordb # or falkordb-remote or neo4j + +# ── Plugin Configuration ─────────────────────────────────────────────────── +# Required when using docker-compose.plugins.yml or docker-compose.dev.yml + +# Your ingress domain (used by Traefik labels on plugin services) +# DOMAIN=localhost + +# OTEL Plugin — span receiver and processor +# OTEL_RECEIVER_PORT=5317 +# OTEL_FILTER_ROUTES=/health,/metrics,/ping,/favicon.ico + +# Memory Plugin — MCP knowledge server +# CGC_MEMORY_HOST=0.0.0.0 +# CGC_MEMORY_PORT=8766 + +# Xdebug Plugin — DBGp TCP listener (dev only) +# XDEBUG_LISTEN_HOST=0.0.0.0 +# XDEBUG_LISTEN_PORT=9003 +# XDEBUG_DEDUP_CACHE_SIZE=10000 + +# Log level for plugin containers (DEBUG, INFO, WARNING, ERROR) +# LOG_LEVEL=INFO diff --git a/.github/services.json b/.github/services.json new file mode 100644 index 00000000..641915b2 --- /dev/null +++ b/.github/services.json @@ -0,0 +1,34 @@ +[ + { + "name": "cgc-core", + "path": ".", + "dockerfile": "Dockerfile", + "image": "cgc-core", + "health_check": "version", + "description": "CodeGraphContext MCP server core" + }, + { + "name": "cgc-plugin-otel", + "path": "plugins/cgc-plugin-otel", + "dockerfile": "Dockerfile", + "image": "cgc-plugin-otel", + "health_check": "grpc_ping", + "description": "OpenTelemetry span receiver and graph writer" + }, + { + "name": "cgc-plugin-xdebug", + "path": "plugins/cgc-plugin-xdebug", + "dockerfile": "Dockerfile", + "image": "cgc-plugin-xdebug", + "health_check": "tcp_connect", + "description": "Xdebug DBGp call-stack listener" + }, + { + "name": "cgc-plugin-memory", + "path": "plugins/cgc-plugin-memory", + "dockerfile": "Dockerfile", + "image": "cgc-plugin-memory", + "health_check": "http_health", + "description": "Project knowledge memory MCP server" + } +] diff --git a/.github/workflows/plugin-publish.yml b/.github/workflows/plugin-publish.yml new file mode 100644 index 00000000..5dfc29b8 --- /dev/null +++ b/.github/workflows/plugin-publish.yml @@ -0,0 +1,127 @@ +name: Build and Publish Plugin Images + +on: + push: + tags: + - 'v*.*.*' + pull_request: + branches: [main] + workflow_dispatch: + +env: + REGISTRY: ghcr.io + IMAGE_PREFIX: ${{ github.repository_owner }} + +jobs: + # ── Read the service matrix from services.json ────────────────────────── + setup: + name: Load service matrix + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.load.outputs.matrix }} + steps: + - uses: actions/checkout@v4 + + - name: Load services.json into matrix + id: load + run: | + # Filter to plugin services only (skip cgc-core — handled by docker-publish.yml) + MATRIX=$(cat .github/services.json | jq -c '[.[] | select(.name != "cgc-core")]') + echo "matrix=${MATRIX}" >> "$GITHUB_OUTPUT" + + # ── Build, smoke-test, and optionally push each plugin image ──────────── + build-plugins: + name: Build ${{ matrix.name }} + needs: setup + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + strategy: + matrix: + include: ${{ fromJson(needs.setup.outputs.matrix) }} + fail-fast: false + + steps: + - uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GHCR + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_PREFIX }}/${{ matrix.image }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=raw,value=latest,enable=${{ startsWith(github.ref, 'refs/tags/') }} + type=ref,event=pr + type=sha,prefix=sha- + labels: | + org.opencontainers.image.title=${{ matrix.name }} + org.opencontainers.image.description=${{ matrix.description }} + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + + - name: Build image (load locally for smoke test) + id: build-local + uses: docker/build-push-action@v5 + with: + context: ${{ matrix.path }} + file: ${{ matrix.path }}/${{ matrix.dockerfile }} + push: false + load: true + tags: smoke-test/${{ matrix.image }}:ci + cache-from: type=gha,scope=${{ matrix.name }} + cache-to: type=gha,mode=max,scope=${{ matrix.name }} + + - name: Smoke test — gRPC import + if: matrix.health_check == 'grpc_ping' + run: docker run --rm smoke-test/${{ matrix.image }}:ci python -c "import grpc; print('gRPC OK')" + + - name: Smoke test — Python import + if: matrix.health_check == 'http_health' + run: docker run --rm smoke-test/${{ matrix.image }}:ci python -c "import cgc_plugin_memory; print('memory OK')" + + - name: Smoke test — socket + if: matrix.health_check == 'tcp_connect' + run: docker run --rm smoke-test/${{ matrix.image }}:ci python -c "import socket; socket.socket(); print('socket OK')" + + - name: Push image to GHCR + if: github.event_name != 'pull_request' + uses: docker/build-push-action@v5 + with: + context: ${{ matrix.path }} + file: ${{ matrix.path }}/${{ matrix.dockerfile }} + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=${{ matrix.name }} + cache-to: type=gha,mode=max,scope=${{ matrix.name }} + platforms: linux/amd64,linux/arm64 + + # ── Summary ────────────────────────────────────────────────────────────── + build-summary: + name: Plugin build summary + needs: build-plugins + runs-on: ubuntu-latest + if: always() + steps: + - name: Report overall status + run: | + if [ "${{ needs.build-plugins.result }}" = "success" ]; then + echo "✅ All plugin images built successfully." + else + echo "⚠️ One or more plugin images failed. Check individual job logs." + exit 1 + fi diff --git a/.github/workflows/test-plugins.yml b/.github/workflows/test-plugins.yml new file mode 100644 index 00000000..8945ece3 --- /dev/null +++ b/.github/workflows/test-plugins.yml @@ -0,0 +1,84 @@ +name: Plugin Tests + +on: + pull_request: + branches: [main] + paths: + - 'plugins/**' + - 'src/codegraphcontext/plugin_registry.py' + - 'tests/unit/plugin/**' + - 'tests/integration/plugin/**' + push: + branches: [main] + paths: + - 'plugins/**' + - 'src/codegraphcontext/plugin_registry.py' + workflow_dispatch: + +jobs: + plugin-unit-tests: + name: Plugin unit + integration tests + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: pip + + - name: Install core CGC (no extras) and dev dependencies + run: | + pip install --no-cache-dir packaging pytest pytest-mock + pip install --no-cache-dir -e ".[dev]" || pip install --no-cache-dir packaging pytest pytest-mock + + - name: Install stub plugin (editable) + run: pip install --no-cache-dir -e plugins/cgc-plugin-stub + + - name: Run plugin unit tests + env: + PYTHONPATH: src + run: pytest tests/unit/plugin/ -v --tb=short + + - name: Run plugin integration tests + env: + PYTHONPATH: src + run: pytest tests/integration/plugin/ -v --tb=short + + plugin-import-check: + name: Verify plugin packages import cleanly + runs-on: ubuntu-latest + strategy: + matrix: + plugin: [cgc-plugin-stub, cgc-plugin-otel, cgc-plugin-xdebug, cgc-plugin-memory] + fail-fast: false + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install plugin + run: | + pip install --no-cache-dir typer neo4j packaging || true + pip install --no-cache-dir -e plugins/${{ matrix.plugin }} || true + + - name: Check plugin PLUGIN_METADATA + env: + PYTHONPATH: src + run: | + PLUGIN_MOD=$(echo "${{ matrix.plugin }}" | tr '-' '_') + python -c " + import importlib + mod = importlib.import_module('${PLUGIN_MOD}') + meta = getattr(mod, 'PLUGIN_METADATA', None) + assert meta is not None, 'PLUGIN_METADATA missing' + for field in ('name', 'version', 'cgc_version_constraint', 'description'): + assert field in meta, f'PLUGIN_METADATA missing field: {field}' + print(f'✅ ${PLUGIN_MOD} PLUGIN_METADATA OK: {meta[\"name\"]} v{meta[\"version\"]}') + " diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..06126001 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,77 @@ +# CodeGraphContext Development Guidelines + +Auto-generated from all feature plans. Last updated: 2026-03-14 + +## Active Technologies + +- Python 3.10+ (constitutional constraint) (001-cgc-plugin-extension) + +## Project Structure + +```text +src/ + codegraphcontext/ + plugin_registry.py ← PluginRegistry (discovers cgc_cli_plugins + cgc_mcp_plugins entry points) + cli/main.py ← CLI app; loads plugin CLI commands at import time + server.py ← MCPServer; loads plugin MCP tools at init time +tests/ + unit/plugin/ ← Unit tests for plugin system (mocked entry points) + integration/plugin/ ← Integration tests (real stub plugin if installed) + e2e/plugin/ ← Full lifecycle E2E tests +plugins/ + cgc-plugin-stub/ ← Reference stub plugin (minimal test fixture) + cgc-plugin-otel/ ← OpenTelemetry span receiver plugin + cgc-plugin-xdebug/ ← Xdebug DBGp call-stack listener plugin + cgc-plugin-memory/ ← Project knowledge memory plugin +docs/ + plugins/ + authoring-guide.md ← How to write a CGC plugin + cross-layer-queries.md ← Canonical cross-layer Cypher queries +``` + +## Commands + +cd src [ONLY COMMANDS FOR ACTIVE TECHNOLOGIES][ONLY COMMANDS FOR ACTIVE TECHNOLOGIES] pytest [ONLY COMMANDS FOR ACTIVE TECHNOLOGIES][ONLY COMMANDS FOR ACTIVE TECHNOLOGIES] ruff check . + +## Code Style + +Python 3.10+ (constitutional constraint): Follow standard conventions + +## Recent Changes + +- 001-cgc-plugin-extension: Added Python 3.10+ (constitutional constraint) + + +## Plugin System (001-cgc-plugin-extension) + +### Entry-point groups +- `cgc_cli_plugins` — plugins contribute a `(name, typer.Typer)` via `get_plugin_commands()` +- `cgc_mcp_plugins` — plugins contribute MCP tools via `get_mcp_tools()` and `get_mcp_handlers()` + +### Plugin layout convention +``` +plugins/cgc-plugin-/ +├── pyproject.toml ← entry-points in both cgc_cli_plugins + cgc_mcp_plugins +└── src/cgc_plugin_/ + ├── __init__.py ← PLUGIN_METADATA dict (required) + ├── cli.py ← get_plugin_commands() + └── mcp_tools.py ← get_mcp_tools() + get_mcp_handlers() +``` + +### MCP tool naming +Plugin tools must be prefixed with plugin name: `_` (e.g. `otel_query_spans`). + +### Install plugins for development +```bash +pip install -e plugins/cgc-plugin-stub # minimal test fixture +pip install -e plugins/cgc-plugin-otel +pip install -e plugins/cgc-plugin-xdebug +pip install -e plugins/cgc-plugin-memory +``` + +### Run plugin tests +```bash +PYTHONPATH=src pytest tests/unit/plugin/ tests/integration/plugin/ -v +PYTHONPATH=src pytest tests/e2e/plugin/ -v # e2e (needs plugins installed) +``` + diff --git a/cgc-extended-spec.md b/cgc-extended-spec.md new file mode 100644 index 00000000..cf51b136 --- /dev/null +++ b/cgc-extended-spec.md @@ -0,0 +1,770 @@ +# CodeGraphContext-Extended (CGC-X) +## Requirements, Specification & Development Plan + +--- + +## 1. Project Overview + +**CodeGraphContext-Extended (CGC-X)** builds on top of the existing [CodeGraphContext](https://github.com/CodeGraphContext/CodeGraphContext) project, extending it with two additional data ingestion pipelines and bundling all components into a single, cohesive Docker Compose deployment. The result is a unified Neo4j knowledge graph that combines three complementary layers of understanding about a codebase. + +### The Three Layers + +| Layer | Source | What It Tells You | +|---|---|---| +| **Static** | CGC (existing) | Code structure — classes, methods, relationships as written | +| **Runtime** | OTEL + Xdebug (new) | Execution reality — what actually runs, how, across services | +| **Memory** | neo4j-memory MCP (new) | Project knowledge — specs, research, decisions, context | + +### Guiding Principles + +- **Same Neo4j instance** — all three layers share one database, enabling cross-layer queries +- **Non-invasive** — no required changes to target applications beyond standard OTEL instrumentation +- **Composable** — each service is independently useful; the value multiplies when combined +- **Homelab-friendly** — runs behind a reverse proxy (Traefik), k8s-compatible, self-contained + +--- + +## 2. Repository Structure + +``` +cgc-extended/ +├── docker-compose.yml # Full stack +├── docker-compose.dev.yml # Dev overrides (Xdebug enabled) +├── .env.example +├── README.md +│ +├── services/ +│ ├── otel-processor/ # NEW: OTEL span → Neo4j ingestion +│ │ ├── Dockerfile +│ │ ├── src/ +│ │ │ ├── main.py +│ │ │ ├── span_processor.py +│ │ │ ├── neo4j_writer.py +│ │ │ └── schema.py +│ │ └── requirements.txt +│ │ +│ └── xdebug-listener/ # NEW: DBGp server → Neo4j ingestion +│ ├── Dockerfile +│ ├── src/ +│ │ ├── main.py +│ │ ├── dbgp_server.py +│ │ ├── neo4j_writer.py +│ │ └── schema.py +│ └── requirements.txt +│ +├── config/ +│ ├── otel-collector/ +│ │ └── config.yaml # OTel Collector pipeline config +│ └── neo4j/ +│ └── init.cypher # Schema constraints & indexes +│ +└── docs/ + ├── neo4j-schema.md + ├── laravel-setup.md + └── traefik-setup.md +``` + +--- + +## 3. Architecture + +``` +┌─────────────────────────────────────────────────────┐ +│ Target Applications │ +│ │ +│ Laravel App A Laravel App B │ +│ (OTEL SDK) (OTEL SDK + Xdebug) │ +└──────┬───────────────────────┬──────────────────────┘ + │ OTLP (gRPC/HTTP) │ OTLP + DBGp (9003) + ▼ │ +┌──────────────┐ │ +│ OTel │ │ +│ Collector │ │ +└──────┬───────┘ │ + │ OTLP (forwarded) │ + ▼ ▼ +┌────────────────────────────────────────────────────┐ +│ CGC-Extended Stack │ +│ │ +│ ┌─────────────────┐ ┌──────────────────────┐ │ +│ │ otel-processor │ │ xdebug-listener │ │ +│ │ (Python) │ │ (Python, port 9003) │ │ +│ └────────┬────────┘ └──────────┬───────────┘ │ +│ │ │ │ +│ ┌────────▼────────────────────────▼───────────┐ │ +│ │ Neo4j │ │ +│ │ (shared with CGC static nodes) │ │ +│ └─────────────────────────────────────────────┘ │ +│ │ │ +│ ┌────────▼────────┐ ┌──────────────────────┐ │ +│ │ CodeGraphCtx │ │ neo4j-memory MCP │ │ +│ │ (CGC, static) │ │ (specs/research) │ │ +│ └─────────────────┘ └──────────────────────┘ │ +└────────────────────────────────────────────────────┘ + │ + ▼ + Traefik (reverse proxy) + → cgc-x.your-domain.com/mcp + → memory.your-domain.com/mcp +``` + +--- + +## 4. Neo4j Unified Schema + +All nodes carry a `source` property that identifies their origin. This is the key to cross-layer querying. + +### Node Labels + +```cypher +// ── STATIC LAYER (CGC existing) ────────────────────────── +(:File { path, language, repo, indexed_at }) +(:Class { name, fqn, file_path, source: 'static' }) +(:Method { name, fqn, file_path, line, source: 'static' }) +(:Function { name, fqn, file_path, line, source: 'static' }) +(:Interface { name, fqn, source: 'static' }) + +// ── RUNTIME LAYER (OTEL) ───────────────────────────────── +(:Service { name, version, environment }) +(:Trace { trace_id, root_span_id, started_at, duration_ms }) +(:Span { + span_id, + trace_id, + name, + service, + kind, // SERVER, CLIENT, INTERNAL, PRODUCER, CONSUMER + class_name, // extracted from span attributes + method_name, // extracted from span attributes + http_method, // for HTTP spans + http_route, // for HTTP spans + db_statement, // for DB spans + duration_ms, + status, + source: 'runtime_otel' +}) + +// ── RUNTIME LAYER (Xdebug) ─────────────────────────────── +(:StackFrame { + class_name, + method_name, + fqn, + file_path, + line, + depth, + source: 'runtime_xdebug' +}) + +// ── MEMORY LAYER (neo4j-memory MCP) ────────────────────── +(:Memory { + id, + name, + entity_type, // spec, decision, research, bug, feature, etc. + created_at, + updated_at, + source: 'memory' +}) +(:Observation { content, created_at }) +``` + +### Relationship Types + +```cypher +// Static +(Method)-[:BELONGS_TO]->(Class) +(Class)-[:IMPLEMENTS]->(Interface) +(Class)-[:EXTENDS]->(Class) +(Method)-[:CALLS]->(Method) +(File)-[:CONTAINS]->(Class) + +// Runtime — OTEL +(Span)-[:CHILD_OF]->(Span) +(Span)-[:PART_OF]->(Trace) +(Trace)-[:ORIGINATED_FROM]->(Service) +(Span)-[:CALLS_SERVICE]->(Service) // cross-service edges + +// Runtime — Xdebug +(StackFrame)-[:CALLED_BY]->(StackFrame) +(StackFrame)-[:RESOLVES_TO]->(Method) // ← links to static layer + +// Memory +(Memory)-[:HAS_OBSERVATION]->(Observation) +(Memory)-[:RELATES_TO]->(Memory) +(Memory)-[:DESCRIBES]->(Class) // ← links to static layer +(Memory)-[:DESCRIBES]->(Method) // ← links to static layer +(Memory)-[:COVERS]->(Span) // ← links to runtime layer + +// Cross-layer correlation +(Span)-[:CORRELATES_TO]->(Method) // OTEL span → static method node +``` + +### Indexes & Constraints + +```cypher +-- init.cypher +CREATE CONSTRAINT class_fqn IF NOT EXISTS + FOR (c:Class) REQUIRE c.fqn IS UNIQUE; + +CREATE CONSTRAINT method_fqn IF NOT EXISTS + FOR (m:Method) REQUIRE m.fqn IS UNIQUE; + +CREATE CONSTRAINT span_id IF NOT EXISTS + FOR (s:Span) REQUIRE s.span_id IS UNIQUE; + +CREATE INDEX span_trace IF NOT EXISTS + FOR (s:Span) ON (s.trace_id); + +CREATE INDEX span_class IF NOT EXISTS + FOR (s:Span) ON (s.class_name); + +CREATE FULLTEXT INDEX memory_search IF NOT EXISTS + FOR (m:Memory) ON EACH [m.name, m.entity_type]; + +CREATE FULLTEXT INDEX observation_search IF NOT EXISTS + FOR (o:Observation) ON EACH [o.content]; +``` + +--- + +## 5. Service Specifications + +### 5.1 OTEL Collector (config only, standard image) + +No custom code — use `otel/opentelemetry-collector-contrib`. + +```yaml +# config/otel-collector/config.yaml +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + timeout: 5s + send_batch_size: 512 + filter/drop_health: # drop noisy health check spans + spans: + exclude: + match_type: strict + attributes: + - key: http.route + value: /health + +exporters: + otlp/processor: # forward to your otel-processor + endpoint: otel-processor:5317 + tls: + insecure: true + +service: + pipelines: + traces: + receivers: [otlp] + processors: [batch, filter/drop_health] + exporters: [otlp/processor] +``` + +**Why a collector?** Decouples the app from your processor. Handles batching, retries, and filtering before spans hit Neo4j. Standard practice — apps just point `OTEL_EXPORTER_OTLP_ENDPOINT` at the collector. + +--- + +### 5.2 OTEL Processor Service + +**Language:** Python +**Base image:** `python:3.12-slim` +**Port:** 5317 (OTLP gRPC, internal only) + +#### Responsibilities + +1. Receive spans from the OTel Collector via OTLP +2. Extract structured data (service name, class, method, HTTP route, DB queries) +3. Upsert nodes and relationships into Neo4j +4. Attempt correlation with CGC static nodes by matching `fqn` + +#### Key Extraction Logic + +Laravel/PHP OTEL spans carry attributes you can parse: + +```python +# span_processor.py + +def extract_php_context(span) -> dict: + attrs = span.attributes or {} + + # Laravel auto-instrumentation sets these + code_namespace = attrs.get('code.namespace', '') # e.g. App\Http\Controllers\OrderController + code_function = attrs.get('code.function', '') # e.g. store + http_route = attrs.get('http.route', '') # e.g. /api/orders + db_statement = attrs.get('db.statement', '') + db_system = attrs.get('db.system', '') + + fqn = f"{code_namespace}::{code_function}" if code_namespace else None + + return { + 'class_name': code_namespace, + 'method_name': code_function, + 'fqn': fqn, + 'http_route': http_route, + 'db_statement': db_statement, + 'db_system': db_system, + } + +def correlate_to_static(tx, span_id: str, fqn: str): + """ + If CGC has already indexed a Method node with this fqn, + draw a CORRELATES_TO edge from the Span to that Method. + """ + tx.run(""" + MATCH (s:Span {span_id: $span_id}) + MATCH (m:Method {fqn: $fqn}) + MERGE (s)-[:CORRELATES_TO]->(m) + """, span_id=span_id, fqn=fqn) +``` + +#### Cross-Service Edge Detection + +When a span has `kind = CLIENT` and `http.url` or `peer.service` set, create a `CALLS_SERVICE` relationship — this is your cross-project graph edge. + +```python +def handle_cross_service(tx, span, context): + if span.kind == SpanKind.CLIENT: + peer = span.attributes.get('peer.service') or \ + extract_host(span.attributes.get('http.url', '')) + if peer: + tx.run(""" + MERGE (target:Service {name: $peer}) + WITH target + MATCH (s:Span {span_id: $span_id}) + MERGE (s)-[:CALLS_SERVICE]->(target) + """, peer=peer, span_id=span.span_id) +``` + +--- + +### 5.3 Xdebug Listener Service + +**Language:** Python +**Base image:** `python:3.12-slim` +**Port:** 9003 (DBGp, exposed — target dev apps connect to this) +**When to run:** Dev/staging only (excluded from production compose) + +#### Responsibilities + +1. Run a DBGp TCP server on port 9003 +2. Accept Xdebug connections from PHP applications +3. Walk stack frames on each breakpoint/trace event +4. Upsert `StackFrame` nodes and `CALLED_BY` edges +5. Attempt `RESOLVES_TO` correlation to CGC `Method` nodes + +#### DBGp Protocol Basics + +``` +PHP (Xdebug client) ──connects to──> DBGp Server (your listener) + +Key commands: + run → continue execution + stack_get → get current call stack (all frames) + context_get → get variables at a given depth +``` + +#### Recommended Library + +Use `python-dbgp` or implement a minimal DBGp server — the protocol is XML over TCP and straightforward: + +```python +# dbgp_server.py (simplified) +import socket, xml.etree.ElementTree as ET + +class DBGpServer: + def handle_connection(self, conn): + # 1. Receive init packet from Xdebug + init = self.recv_packet(conn) + + # 2. Send `run` to let execution proceed to next breakpoint + self.send_cmd(conn, 'run') + + # 3. On each stop, fetch the full stack + while True: + response = self.recv_packet(conn) + if response is None: + break + + self.send_cmd(conn, 'stack_get -i 1') + stack_xml = self.recv_packet(conn) + frames = self.parse_stack(stack_xml) + + self.write_to_neo4j(frames) + self.send_cmd(conn, 'run') + + def parse_stack(self, xml_str) -> list[dict]: + root = ET.fromstring(xml_str) + frames = [] + for stack in root.findall('stack'): + frames.append({ + 'class': stack.get('classname', ''), + 'method': stack.get('where', ''), + 'file': stack.get('filename', ''), + 'line': int(stack.get('lineno', 0)), + 'depth': int(stack.get('level', 0)), + }) + return frames +``` + +#### Deduplication Strategy + +The same call chain will repeat across thousands of requests. Use a hash of the call chain to deduplicate: + +```python +import hashlib + +def chain_hash(frames: list[dict]) -> str: + key = '|'.join(f"{f['class']}::{f['method']}" for f in frames) + return hashlib.sha256(key.encode()).hexdigest()[:16] + +# In neo4j_writer: only upsert if hash not seen recently +# Keep a local LRU cache of recent chain hashes to avoid Neo4j round-trips +``` + +--- + +### 5.4 Memory MCP Service + +Use the official `mcp/neo4j-memory` Docker image. No custom code required. + +**Configuration:** +```yaml +# docker-compose.yml excerpt +cgc-memory: + image: mcp/neo4j-memory + environment: + NEO4J_URL: bolt://neo4j:7687 + NEO4J_USERNAME: neo4j + NEO4J_PASSWORD: ${NEO4J_PASSWORD} + NEO4J_DATABASE: neo4j # same DB as everything else + NEO4J_MCP_SERVER_HOST: 0.0.0.0 + NEO4J_MCP_SERVER_PORT: 8766 +``` + +**Usage guidance for your team:** + +Store the following entity types to get maximum value: +- `spec` — functional requirements, acceptance criteria +- `decision` — architectural decisions with rationale (lightweight ADR) +- `research` — spike findings, library evaluations +- `bug` — known issues, reproduction steps, root cause once found +- `feature` — planned work with context +- `integration` — notes on cross-service contracts and dependencies + +When a Memory node `DESCRIBES` a Class or Method that CGC has indexed, the AI assistant can answer questions like: *"Show me the spec for the payment service and which methods implement it."* + +--- + +## 6. Docker Compose + +```yaml +# docker-compose.yml +services: + + neo4j: + image: neo4j:5 + container_name: cgc-neo4j + restart: unless-stopped + environment: + NEO4J_AUTH: neo4j/${NEO4J_PASSWORD} + NEO4J_PLUGINS: '["apoc"]' + NEO4J_dbms_memory_heap_max__size: 2G + volumes: + - neo4j_data:/data + - ./config/neo4j/init.cypher:/var/lib/neo4j/import/init.cypher + ports: + - "7687:7687" # Bolt (internal use) + - "7474:7474" # Browser (optional, disable in prod) + healthcheck: + test: ["CMD", "neo4j", "status"] + interval: 30s + timeout: 10s + retries: 5 + + codegraphcontext: + image: codegraphcontext/codegraphcontext:latest # or build from source + container_name: cgc-static + restart: unless-stopped + environment: + NEO4J_URI: bolt://neo4j:7687 + NEO4J_USERNAME: neo4j + NEO4J_PASSWORD: ${NEO4J_PASSWORD} + depends_on: + neo4j: + condition: service_healthy + labels: + - "traefik.enable=true" + - "traefik.http.routers.cgc.rule=Host(`cgc.${DOMAIN}`)" + + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + container_name: cgc-otel-collector + restart: unless-stopped + volumes: + - ./config/otel-collector/config.yaml:/etc/otelcol-contrib/config.yaml + ports: + - "4317:4317" # OTLP gRPC (apps send here) + - "4318:4318" # OTLP HTTP (apps send here) + depends_on: + - otel-processor + + otel-processor: + build: ./services/otel-processor + container_name: cgc-otel-processor + restart: unless-stopped + environment: + NEO4J_URI: bolt://neo4j:7687 + NEO4J_USERNAME: neo4j + NEO4J_PASSWORD: ${NEO4J_PASSWORD} + LISTEN_PORT: 5317 + LOG_LEVEL: INFO + depends_on: + neo4j: + condition: service_healthy + + cgc-memory: + image: mcp/neo4j-memory + container_name: cgc-memory + restart: unless-stopped + environment: + NEO4J_URL: bolt://neo4j:7687 + NEO4J_USERNAME: neo4j + NEO4J_PASSWORD: ${NEO4J_PASSWORD} + NEO4J_DATABASE: neo4j + NEO4J_MCP_SERVER_HOST: 0.0.0.0 + NEO4J_MCP_SERVER_PORT: 8766 + depends_on: + neo4j: + condition: service_healthy + labels: + - "traefik.enable=true" + - "traefik.http.routers.cgc-memory.rule=Host(`memory.${DOMAIN}`)" + +volumes: + neo4j_data: +``` + +```yaml +# docker-compose.dev.yml (override for development) +services: + xdebug-listener: + build: ./services/xdebug-listener + container_name: cgc-xdebug + restart: unless-stopped + environment: + NEO4J_URI: bolt://neo4j:7687 + NEO4J_USERNAME: neo4j + NEO4J_PASSWORD: ${NEO4J_PASSWORD} + LISTEN_HOST: 0.0.0.0 + LISTEN_PORT: 9003 + DEDUP_CACHE_SIZE: 10000 + LOG_LEVEL: DEBUG + ports: + - "9003:9003" # DBGp — PHP apps connect to this + depends_on: + neo4j: + condition: service_healthy +``` + +--- + +## 7. Laravel Application Setup + +### OTEL Instrumentation (Production + Dev) + +```bash +composer require \ + open-telemetry/sdk \ + open-telemetry/exporter-otlp \ + open-telemetry/opentelemetry-auto-laravel \ + open-telemetry/opentelemetry-auto-psr18 +``` + +Add to `.env`: +```ini +OTEL_PHP_AUTOLOAD_ENABLED=true +OTEL_SERVICE_NAME=your-service-name +OTEL_TRACES_EXPORTER=otlp +OTEL_EXPORTER_OTLP_PROTOCOL=grpc +OTEL_EXPORTER_OTLP_ENDPOINT=http://cgc-otel-collector:4317 +OTEL_PROPAGATORS=tracecontext,baggage +OTEL_TRACES_SAMPLER=parentbased_traceidratio +OTEL_TRACES_SAMPLER_ARG=1.0 # 1.0 = 100% in dev, lower in prod +``` + +Add to `Dockerfile`: +```dockerfile +RUN pecl install opentelemetry +RUN echo "extension=opentelemetry.so" >> /usr/local/etc/php/conf.d/opentelemetry.ini +``` + +### Xdebug (Dev only) + +```dockerfile +# dev.Dockerfile or override +RUN pecl install xdebug +``` + +```ini +; xdebug.ini +xdebug.mode=debug,trace +xdebug.client_host=cgc-xdebug ; container name in same Docker network +xdebug.client_port=9003 +xdebug.start_with_request=trigger ; use XDEBUG_TRIGGER header/cookie +; or: xdebug.start_with_request=yes for all requests (noisy) +``` + +**Recommended:** use `trigger` mode. Set the `XDEBUG_TRIGGER` cookie in your browser to selectively capture traces rather than flooding Neo4j on every request. + +--- + +## 8. Development Phases + +### Phase 1 — Foundation (Week 1–2) + +Goal: Neo4j running, CGC indexing, schema in place. + +- [ ] Set up repository structure +- [ ] Write `config/neo4j/init.cypher` with constraints and indexes +- [ ] Wire up `docker-compose.yml` with Neo4j + CGC + memory MCP +- [ ] Verify CGC indexes a Laravel project into Neo4j +- [ ] Verify `mcp/neo4j-memory` connects to same DB and nodes are queryable +- [ ] Set up Traefik labels and confirm both MCP endpoints are accessible +- [ ] Write `docs/neo4j-schema.md` as living document + +**Success criterion:** AI assistant can query static code nodes AND store/retrieve memory entities, in the same Neo4j instance. + +--- + +### Phase 2 — OTEL Processor (Week 2–3) + +Goal: Laravel spans flowing into Neo4j, basic cross-layer correlation working. + +- [ ] Scaffold `services/otel-processor/` — Python OTLP receiver +- [ ] Implement span → Neo4j upsert for `Span`, `Trace`, `Service` nodes +- [ ] Implement `CHILD_OF` relationship from `parent_span_id` +- [ ] Implement PHP attribute extraction (`code.namespace`, `code.function`) +- [ ] Implement `CORRELATES_TO` correlation against existing CGC `Method` nodes +- [ ] Implement cross-service edge detection (`SpanKind.CLIENT`) +- [ ] Wire `otel-collector` → `otel-processor` in compose +- [ ] Test with a real Laravel app: instrument, send request, verify nodes appear +- [ ] Write `docs/laravel-setup.md` + +**Success criterion:** A single HTTP request to the Laravel app produces a complete span tree in Neo4j, with at least some spans connected to static Method nodes. + +--- + +### Phase 3 — Xdebug Listener (Week 3–4) + +Goal: Dev-time method-level traces captured and linked to static nodes. + +- [ ] Scaffold `services/xdebug-listener/` — Python DBGp server +- [ ] Implement TCP server, DBGp handshake, `stack_get` command +- [ ] Implement stack frame parsing and `StackFrame` node upsert +- [ ] Implement `CALLED_BY` chain from frame depth +- [ ] Implement call chain deduplication (hash + LRU cache) +- [ ] Implement `RESOLVES_TO` correlation to CGC `Method` nodes by `fqn` +- [ ] Wire into `docker-compose.dev.yml` +- [ ] Test: trigger Xdebug on a Laravel request, verify frame graph in Neo4j + +**Success criterion:** Xdebug trace for a request shows container-resolved classes (e.g., concrete repository implementation rather than interface) connected to the static graph. + +--- + +### Phase 4 — Cross-Layer Queries & MCP Tooling (Week 4–5) + +Goal: The unified graph is queryable in useful ways from an AI assistant. + +**Example queries to validate and document:** + +```cypher +-- "Show me everything that executes when POST /api/orders is called" +MATCH (s:Span {http_route: '/api/orders', http_method: 'POST'}) +MATCH (s)-[:CHILD_OF*1..10]->(child:Span) +OPTIONAL MATCH (child)-[:CORRELATES_TO]->(m:Method) +RETURN s, child, m + +-- "Which specs describe code that was called in the last hour?" +MATCH (mem:Memory)-[:DESCRIBES]->(m:Method) +MATCH (s:Span)-[:CORRELATES_TO]->(m) +WHERE s.started_at > timestamp() - 3600000 +RETURN mem.name, mem.entity_type, m.fqn, s.name + +-- "Show cross-service call chains" +MATCH (svc1:Service)-[:ORIGINATED_FROM]-(t:Trace)-[:PART_OF]-(s:Span) +MATCH (s)-[:CALLS_SERVICE]->(svc2:Service) +RETURN svc1.name, svc2.name, count(*) as call_count +ORDER BY call_count DESC + +-- "What code runs that has no spec?" +MATCH (m:Method)<-[:CORRELATES_TO]-(s:Span) +WHERE NOT EXISTS { MATCH (mem:Memory)-[:DESCRIBES]->(m) } +RETURN m.fqn, count(s) as execution_count +ORDER BY execution_count DESC +``` + +- [ ] Write and test the above canonical queries +- [ ] Document queries in `docs/neo4j-schema.md` +- [ ] Consider a thin MCP wrapper exposing these as named tools (optional) + +--- + +### Phase 5 — Polish & Release (Week 5–6) + +- [ ] Write comprehensive `README.md` with architecture diagram +- [ ] Create `.env.example` with all required variables documented +- [ ] Add `CONTRIBUTING.md` with credit to upstream CGC project +- [ ] Add health check endpoints to both custom services +- [ ] Test full stack teardown and restart (data persistence) +- [ ] Test k8s manifests (port from existing homelab patterns) +- [ ] Tag v0.1.0 + +--- + +## 9. Environment Variables Reference + +```ini +# .env.example + +# Neo4j +NEO4J_PASSWORD=changeme +DOMAIN=yourdomain.local + +# OTEL Processor +OTEL_PROCESSOR_LOG_LEVEL=INFO +OTEL_PROCESSOR_BATCH_SIZE=100 +OTEL_PROCESSOR_FLUSH_INTERVAL=5 + +# Xdebug Listener (dev only) +XDEBUG_DEDUP_CACHE_SIZE=10000 +XDEBUG_MAX_DEPTH=20 # max stack depth to capture + +# Memory MCP +# (uses NEO4J_* vars above, no additional config needed) +``` + +--- + +## 10. Key Design Decisions + +**Why Python for otel-processor and xdebug-listener?** +The `opentelemetry-sdk` Python package has excellent OTLP receiver support and Neo4j's official `neo4j` Python driver is the most mature. Keeps both services consistent. + +**Why same Neo4j database (not separate databases)?** +Cross-layer queries require traversing between node types. If CGC static nodes and OTEL span nodes are in different databases, you cannot do `MATCH (s:Span)-[:CORRELATES_TO]->(m:Method)` in a single query. The unified schema with `source` property labels is sufficient to distinguish origins. + +**Why the OTel Collector in between?** +Direct OTLP from app → otel-processor works but is fragile. The collector handles batching, retry on failure, and gives you a place to add sampling rules or additional exporters (e.g., Jaeger for visual trace inspection) without touching application config. + +**Why `mcp/neo4j-memory` rather than a custom memory service?** +It's maintained, well-documented, and covers the generic memory use case well. The value of CGC-X is the unified graph — not reinventing memory storage. + +**Xdebug `trigger` mode rather than `yes` mode?** +`yes` mode captures every request, generating massive graph noise and degrading performance. `trigger` mode lets you selectively capture specific requests using the `XDEBUG_TRIGGER` cookie/header, giving you targeted, high-quality traces. diff --git a/config/neo4j/init.cypher b/config/neo4j/init.cypher new file mode 100644 index 00000000..4014d75b --- /dev/null +++ b/config/neo4j/init.cypher @@ -0,0 +1,38 @@ +// CGC Plugin Extension — Graph Schema Initialization +// Run this against Neo4j after startup to create all constraints and indexes. +// Idempotent: uses IF NOT EXISTS throughout. + +// ── OTEL Plugin: Service nodes ───────────────────────────────────────────── +CREATE CONSTRAINT service_name IF NOT EXISTS + FOR (s:Service) REQUIRE s.name IS UNIQUE; + +// ── OTEL Plugin: Trace nodes ─────────────────────────────────────────────── +CREATE CONSTRAINT trace_id IF NOT EXISTS + FOR (t:Trace) REQUIRE t.trace_id IS UNIQUE; + +// ── OTEL Plugin: Span nodes ──────────────────────────────────────────────── +CREATE CONSTRAINT span_id IF NOT EXISTS + FOR (s:Span) REQUIRE s.span_id IS UNIQUE; + +CREATE INDEX span_trace IF NOT EXISTS + FOR (s:Span) ON (s.trace_id); + +CREATE INDEX span_class IF NOT EXISTS + FOR (s:Span) ON (s.class_name); + +CREATE INDEX span_route IF NOT EXISTS + FOR (s:Span) ON (s.http_route); + +// ── Xdebug Plugin: StackFrame nodes ─────────────────────────────────────── +CREATE CONSTRAINT frame_id IF NOT EXISTS + FOR (sf:StackFrame) REQUIRE sf.frame_id IS UNIQUE; + +CREATE INDEX frame_fqn IF NOT EXISTS + FOR (sf:StackFrame) ON (sf.fqn); + +// ── Memory Plugin: Memory + Observation nodes ────────────────────────────── +CREATE FULLTEXT INDEX memory_search IF NOT EXISTS + FOR (m:Memory) ON EACH [m.name, m.entity_type]; + +CREATE FULLTEXT INDEX observation_search IF NOT EXISTS + FOR (o:Observation) ON EACH [o.content]; diff --git a/config/otel-collector/config.yaml b/config/otel-collector/config.yaml new file mode 100644 index 00000000..52dcdab2 --- /dev/null +++ b/config/otel-collector/config.yaml @@ -0,0 +1,33 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + +processors: + batch: + timeout: 5s + send_batch_size: 512 + + filter/drop_noise: + error_mode: ignore + traces: + span: + - 'attributes["http.route"] == "/health"' + - 'attributes["http.route"] == "/metrics"' + - 'attributes["http.route"] == "/ping"' + +exporters: + otlp/cgc_processor: + endpoint: cgc-otel-processor:5317 + tls: + insecure: true + +service: + pipelines: + traces: + receivers: [otlp] + processors: [filter/drop_noise, batch] + exporters: [otlp/cgc_processor] diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml new file mode 100644 index 00000000..a8664f3e --- /dev/null +++ b/docker-compose.dev.yml @@ -0,0 +1,38 @@ +# Development overlay — adds Xdebug DBGp listener service. +# +# Usage (with plugin stack — recommended): +# docker compose -f docker-compose.plugin-stack.yml -f docker-compose.dev.yml up -d +# +# Usage (with core template + neo4j profile): +# docker compose -f docker-compose.template.yml --profile neo4j -f docker-compose.dev.yml up -d +# +# IMPORTANT: The Xdebug listener only starts when CGC_PLUGIN_XDEBUG_ENABLED=true. +# IMPORTANT: Requires neo4j service with a healthcheck (provided by docker-compose.plugin-stack.yml). + +version: '3.8' + +services: + + # ── CGC Xdebug DBGp listener ────────────────────────────────────────────── + xdebug-listener: + build: + context: plugins/cgc-plugin-xdebug + dockerfile: Dockerfile + container_name: cgc-xdebug-listener + environment: + - NEO4J_URI=${NEO4J_URI:-bolt://neo4j:7687} + - NEO4J_USERNAME=${NEO4J_USERNAME:-neo4j} + - NEO4J_PASSWORD=${NEO4J_PASSWORD:-codegraph123} + - XDEBUG_LISTEN_HOST=${XDEBUG_LISTEN_HOST:-0.0.0.0} + - XDEBUG_LISTEN_PORT=${XDEBUG_LISTEN_PORT:-9003} + - XDEBUG_DEDUP_CACHE_SIZE=${XDEBUG_DEDUP_CACHE_SIZE:-10000} + - CGC_PLUGIN_XDEBUG_ENABLED=true + - LOG_LEVEL=${LOG_LEVEL:-DEBUG} + ports: + - "9003:9003" + depends_on: + neo4j: + condition: service_healthy + networks: + - cgc-network + restart: unless-stopped diff --git a/docker-compose.plugin-stack.yml b/docker-compose.plugin-stack.yml new file mode 100644 index 00000000..8a04ff8e --- /dev/null +++ b/docker-compose.plugin-stack.yml @@ -0,0 +1,159 @@ +# Full CGC plugin stack — self-contained for local development and manual testing. +# +# Includes: Neo4j + CGC core + OTEL collector + OTEL processor + Memory plugin. +# Add Xdebug listener with: -f docker-compose.dev.yml +# +# Quick start: +# cp .env.example .env # edit NEO4J_PASSWORD at minimum +# docker compose -f docker-compose.plugin-stack.yml up -d +# docker compose -f docker-compose.plugin-stack.yml logs -f +# +# With Xdebug (dev): +# docker compose -f docker-compose.plugin-stack.yml -f docker-compose.dev.yml up -d +# +# Verify: +# docker compose -f docker-compose.plugin-stack.yml ps +# curl -s http://localhost:7474 # Neo4j Browser +# grpcurl -plaintext localhost:4317 list # OTEL gRPC endpoint (needs grpcurl) + +version: '3.8' + +services: + + # ── Neo4j graph database ─────────────────────────────────────────────────── + neo4j: + image: neo4j:5.15.0 + container_name: cgc-neo4j + ports: + - "7474:7474" # Browser: http://localhost:7474 + - "7687:7687" # Bolt + environment: + - NEO4J_AUTH=${NEO4J_USERNAME:-neo4j}/${NEO4J_PASSWORD:-codegraph123} + - NEO4J_PLUGINS=["apoc"] + - NEO4J_dbms_security_procedures_unrestricted=apoc.* + - NEO4J_dbms_memory_heap_max__size=2G + volumes: + - neo4j-data:/data + - neo4j-logs:/logs + - ./config/neo4j/init.cypher:/docker-entrypoint-initdb.d/init.cypher:ro + healthcheck: + test: ["CMD-SHELL", "wget -q --spider http://localhost:7474 || exit 1"] + interval: 15s + timeout: 10s + retries: 5 + start_period: 30s + networks: + - cgc-network + restart: unless-stopped + + # ── CGC core MCP server ──────────────────────────────────────────────────── + cgc-core: + build: + context: . + dockerfile: Dockerfile + container_name: cgc-core + environment: + - DATABASE_TYPE=neo4j + - NEO4J_URI=bolt://neo4j:7687 + - NEO4J_USERNAME=${NEO4J_USERNAME:-neo4j} + - NEO4J_PASSWORD=${NEO4J_PASSWORD:-codegraph123} + - PYTHONUNBUFFERED=1 + volumes: + - ./:/workspace + - cgc-data:/root/.codegraphcontext + stdin_open: true + tty: true + depends_on: + neo4j: + condition: service_healthy + networks: + - cgc-network + restart: unless-stopped + + # ── OpenTelemetry Collector ──────────────────────────────────────────────── + # Receives spans from your application (ports 4317 gRPC, 4318 HTTP), + # filters noise, and forwards to cgc-otel-processor. + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + container_name: cgc-otel-collector + command: ["--config=/etc/otelcol/config.yaml"] + volumes: + - ./config/otel-collector/config.yaml:/etc/otelcol/config.yaml:ro + ports: + - "4317:4317" # OTLP gRPC — point your app here + - "4318:4318" # OTLP HTTP — alternative ingestion + depends_on: + - cgc-otel-processor + networks: + - cgc-network + restart: unless-stopped + + # ── CGC OTEL Processor ──────────────────────────────────────────────────── + # Receives filtered spans from collector, writes Service/Trace/Span nodes + # to Neo4j and correlates them to static Method nodes. + cgc-otel-processor: + build: + context: plugins/cgc-plugin-otel + dockerfile: Dockerfile + container_name: cgc-otel-processor + environment: + - NEO4J_URI=bolt://neo4j:7687 + - NEO4J_USERNAME=${NEO4J_USERNAME:-neo4j} + - NEO4J_PASSWORD=${NEO4J_PASSWORD:-codegraph123} + - OTEL_RECEIVER_PORT=${OTEL_RECEIVER_PORT:-5317} + - OTEL_FILTER_ROUTES=${OTEL_FILTER_ROUTES:-/health,/metrics,/ping} + - LOG_LEVEL=${LOG_LEVEL:-INFO} + ports: + - "5317:5317" # Internal gRPC (collector → processor; not exposed to app) + depends_on: + neo4j: + condition: service_healthy + networks: + - cgc-network + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "python -c \"import grpc; print('ok')\" || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 15s + + # ── CGC Memory MCP server ───────────────────────────────────────────────── + # Stores and retrieves project knowledge (specs, notes, ADRs) linked to + # static code nodes in the graph. + cgc-memory: + build: + context: plugins/cgc-plugin-memory + dockerfile: Dockerfile + container_name: cgc-memory + environment: + - NEO4J_URI=bolt://neo4j:7687 + - NEO4J_USERNAME=${NEO4J_USERNAME:-neo4j} + - NEO4J_PASSWORD=${NEO4J_PASSWORD:-codegraph123} + - NEO4J_DATABASE=${NEO4J_DATABASE:-neo4j} + - CGC_MEMORY_HOST=${CGC_MEMORY_HOST:-0.0.0.0} + - CGC_MEMORY_PORT=${CGC_MEMORY_PORT:-8766} + - LOG_LEVEL=${LOG_LEVEL:-INFO} + ports: + - "8766:8766" # MCP server + depends_on: + neo4j: + condition: service_healthy + networks: + - cgc-network + restart: unless-stopped + healthcheck: + test: ["CMD-SHELL", "python -c \"import cgc_plugin_memory; print('ok')\" || exit 1"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 10s + +volumes: + neo4j-data: + neo4j-logs: + cgc-data: + +networks: + cgc-network: + driver: bridge diff --git a/docker-compose.plugins.yml b/docker-compose.plugins.yml new file mode 100644 index 00000000..623fee02 --- /dev/null +++ b/docker-compose.plugins.yml @@ -0,0 +1,86 @@ +# Plugin services overlay — OTEL collector/processor + Memory MCP server. +# +# NOTE: For local development, prefer docker-compose.plugin-stack.yml which is +# self-contained and includes Neo4j with a healthcheck. +# +# Usage (overlay on plugin-stack — recommended for adding to existing stack): +# # Already included in docker-compose.plugin-stack.yml +# +# Usage (overlay on core template — requires neo4j profile active): +# docker compose -f docker-compose.template.yml --profile neo4j -f docker-compose.plugins.yml up +# +# Prerequisites: +# - neo4j service running with healthcheck (docker-compose.plugin-stack.yml provides this) +# - NEO4J_URI, NEO4J_USERNAME, NEO4J_PASSWORD set in .env +# - DOMAIN set to your ingress domain (e.g. localhost) + +version: '3.8' + +services: + + # ── OpenTelemetry Collector ─────────────────────────────────────────────── + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + container_name: cgc-otel-collector + command: ["--config=/etc/otelcol/config.yaml"] + volumes: + - ./config/otel-collector/config.yaml:/etc/otelcol/config.yaml:ro + ports: + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP + depends_on: + - cgc-otel-processor + networks: + - cgc-network + restart: unless-stopped + + # ── CGC OTEL Processor (receives from collector, writes to Neo4j) ───────── + cgc-otel-processor: + build: + context: plugins/cgc-plugin-otel + dockerfile: Dockerfile + container_name: cgc-otel-processor + environment: + - NEO4J_URI=${NEO4J_URI:-bolt://neo4j:7687} + - NEO4J_USERNAME=${NEO4J_USERNAME:-neo4j} + - NEO4J_PASSWORD=${NEO4J_PASSWORD:-codegraph123} + - OTEL_RECEIVER_PORT=${OTEL_RECEIVER_PORT:-5317} + - OTEL_FILTER_ROUTES=${OTEL_FILTER_ROUTES:-/health,/metrics,/ping} + - LOG_LEVEL=${LOG_LEVEL:-INFO} + ports: + - "5317:5317" # internal gRPC (collector → processor) + depends_on: + neo4j: + condition: service_healthy + networks: + - cgc-network + restart: unless-stopped + labels: + - "traefik.enable=true" + - "traefik.http.routers.otel.rule=Host(`otel.${DOMAIN:-localhost}`)" + + # ── CGC Memory MCP server ───────────────────────────────────────────────── + cgc-memory: + build: + context: plugins/cgc-plugin-memory + dockerfile: Dockerfile + container_name: cgc-memory + environment: + - NEO4J_URI=${NEO4J_URI:-bolt://neo4j:7687} + - NEO4J_USERNAME=${NEO4J_USERNAME:-neo4j} + - NEO4J_PASSWORD=${NEO4J_PASSWORD:-codegraph123} + - NEO4J_DATABASE=${NEO4J_DATABASE:-neo4j} + - CGC_MEMORY_HOST=${CGC_MEMORY_HOST:-0.0.0.0} + - CGC_MEMORY_PORT=${CGC_MEMORY_PORT:-8766} + - LOG_LEVEL=${LOG_LEVEL:-INFO} + ports: + - "8766:8766" + depends_on: + neo4j: + condition: service_healthy + networks: + - cgc-network + restart: unless-stopped + labels: + - "traefik.enable=true" + - "traefik.http.routers.memory.rule=Host(`memory.${DOMAIN:-localhost}`)" diff --git a/docker-compose.template.yml b/docker-compose.template.yml index 7c406e3d..80a79488 100644 --- a/docker-compose.template.yml +++ b/docker-compose.template.yml @@ -41,24 +41,32 @@ services: - falkordb # Optional: Neo4j database (if you prefer Neo4j over FalkorDB) + # Required when using any CGC plugin (otel, memory, xdebug). neo4j: image: neo4j:5.15.0 container_name: cgc-neo4j ports: - - "7474:7474" # HTTP + - "7474:7474" # HTTP Browser - "7687:7687" # Bolt environment: - - NEO4J_AUTH=neo4j/codegraph123 + - NEO4J_AUTH=${NEO4J_USERNAME:-neo4j}/${NEO4J_PASSWORD:-codegraph123} - NEO4J_PLUGINS=["apoc"] - NEO4J_dbms_security_procedures_unrestricted=apoc.* - NEO4J_dbms_memory_heap_max__size=2G volumes: - neo4j-data:/data - neo4j-logs:/logs + - ./config/neo4j/init.cypher:/docker-entrypoint-initdb.d/init.cypher:ro + healthcheck: + test: ["CMD-SHELL", "wget -q --spider http://localhost:7474 || exit 1"] + interval: 15s + timeout: 10s + retries: 5 + start_period: 30s networks: - cgc-network profiles: - - neo4j # Only start when explicitly requested + - neo4j # Start with: docker compose --profile neo4j up volumes: cgc-data: diff --git a/docs/plugins/authoring-guide.md b/docs/plugins/authoring-guide.md new file mode 100644 index 00000000..f0bb910f --- /dev/null +++ b/docs/plugins/authoring-guide.md @@ -0,0 +1,226 @@ +# Plugin Authoring Guide + +This guide walks through creating a CGC plugin from scratch. +The `plugins/cgc-plugin-stub` directory is the canonical worked example — reference it +throughout. + +For the full contract specification see: +[`specs/001-cgc-plugin-extension/contracts/plugin-interface.md`](../../specs/001-cgc-plugin-extension/contracts/plugin-interface.md) + +--- + +## 1. Package Scaffold + +A CGC plugin is a standard Python package with two entry-point groups. + +``` +plugins/cgc-plugin-/ +├── pyproject.toml +└── src/ + └── cgc_plugin_/ + ├── __init__.py ← PLUGIN_METADATA + re-exports + ├── cli.py ← get_plugin_commands() + └── mcp_tools.py ← get_mcp_tools() + get_mcp_handlers() +``` + +Bootstrap it by copying the stub: + +```bash +cp -r plugins/cgc-plugin-stub plugins/cgc-plugin-myname +# then edit: pyproject.toml, __init__.py, cli.py, mcp_tools.py +``` + +--- + +## 2. `pyproject.toml` + +Minimum required configuration: + +```toml +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "cgc-plugin-myname" +version = "0.1.0" +requires-python = ">=3.10" +dependencies = ["typer[all]>=0.9.0"] + +[project.entry-points.cgc_cli_plugins] +myname = "cgc_plugin_myname" + +[project.entry-points.cgc_mcp_plugins] +myname = "cgc_plugin_myname" +``` + +**Key points**: +- Entry point group: `cgc_cli_plugins` — for CLI commands +- Entry point group: `cgc_mcp_plugins` — for MCP tools +- Entry point name (`myname`) becomes the CLI command group name and the registry key +- Both groups must point to the same module for most plugins + +--- + +## 3. `__init__.py` — PLUGIN_METADATA + +```python +PLUGIN_METADATA = { + "name": "cgc-plugin-myname", # must match pyproject.toml name + "version": "0.1.0", + "cgc_version_constraint": ">=0.1.0", # PEP 440 specifier + "description": "One-line description of what this plugin does", +} +``` + +**Required fields**: `name`, `version`, `cgc_version_constraint`, `description`. +Missing any field causes the plugin to be skipped at startup with a clear warning. + +The `cgc_version_constraint` is checked against the installed `codegraphcontext` version. +Use `">=0.1.0"` for maximum compatibility during early development. + +--- + +## 4. CLI Contract — `cli.py` + +```python +import typer + +myname_app = typer.Typer(help="My plugin commands.") + +@myname_app.command("hello") +def hello(name: str = typer.Option("World", help="Name to greet")): + """Say hello from myname plugin.""" + typer.echo(f"Hello from myname plugin, {name}!") + + +def get_plugin_commands(): + """Return (command_group_name, typer_app) to be registered with CGC.""" + return ("myname", myname_app) +``` + +**Contract**: +- `get_plugin_commands()` must return a `(str, typer.Typer)` tuple +- The string becomes the sub-command group: `cgc myname ` +- Raising an exception in `get_plugin_commands()` quarantines the plugin safely + +--- + +## 5. MCP Contract — `mcp_tools.py` + +```python +def get_mcp_tools(server_context: dict | None = None): + """Return dict of tool_name → MCP tool definition.""" + return { + "myname_hello": { + "name": "myname_hello", + "description": "Say hello from myname plugin", + "inputSchema": { + "type": "object", + "properties": { + "name": {"type": "string", "description": "Name to greet"}, + }, + "required": ["name"], + }, + }, + } + + +def get_mcp_handlers(server_context: dict | None = None): + """Return dict of tool_name → callable handler.""" + db = (server_context or {}).get("db_manager") + + def handle_hello(name: str = "World"): + return {"greeting": f"Hello {name} from myname plugin!"} + + return {"myname_hello": handle_hello} +``` + +**Contract**: +- `get_mcp_tools()` returns `dict[str, ToolDefinition]` +- `get_mcp_handlers()` returns `dict[str, callable]` +- Tool names **must** be prefixed with the plugin name: `_` +- `server_context` carries `{"db_manager": }` when available +- Conflicting tool names: the first plugin to register a name wins + +--- + +## 6. Accessing Neo4j + +If your plugin needs graph access, use the `db_manager` from `server_context`: + +```python +def get_mcp_handlers(server_context=None): + db = (server_context or {}).get("db_manager") + + def handle_query(limit: int = 10): + if db is None: + return {"error": "No database connection available"} + results = db.execute_query( + "MATCH (n:Method) RETURN n.fqn LIMIT $limit", + {"limit": limit} + ) + return {"methods": [r["n.fqn"] for r in results]} + + return {"myname_query": handle_query} +``` + +--- + +## 7. Testing Your Plugin + +Write tests in `tests/unit/plugin/` and `tests/integration/plugin/`. + +```python +# tests/unit/plugin/test_myname_tools.py +from cgc_plugin_myname.mcp_tools import get_mcp_tools, get_mcp_handlers + +def test_tools_defined(): + tools = get_mcp_tools() + assert "myname_hello" in tools + +def test_hello_handler(): + handlers = get_mcp_handlers() + result = handlers["myname_hello"](name="Test") + assert result["greeting"] == "Hello Test from myname plugin!" +``` + +Run tests: +```bash +PYTHONPATH=src pytest tests/unit/plugin/ tests/integration/plugin/ -v +``` + +--- + +## 8. Install and Verify + +```bash +pip install -e plugins/cgc-plugin-myname + +# Verify CLI registration +cgc --help # should show 'myname' group +cgc plugin list # should show cgc-plugin-myname as loaded + +# Verify MCP registration (start MCP server and inspect tools/list) +cgc mcp start +# In MCP client: tools/list → should include myname_hello +``` + +--- + +## 9. Publishing to PyPI + +```bash +cd plugins/cgc-plugin-myname +pip install build +python -m build +pip install twine +twine upload dist/* +``` + +Users then install your plugin with: +```bash +pip install cgc-plugin-myname +``` + +CGC discovers it automatically at next startup — no configuration required. diff --git a/docs/plugins/cross-layer-queries.md b/docs/plugins/cross-layer-queries.md new file mode 100644 index 00000000..50c9ff63 --- /dev/null +++ b/docs/plugins/cross-layer-queries.md @@ -0,0 +1,173 @@ +# Cross-Layer Cypher Queries + +These five canonical queries validate **SC-005** (cross-layer intelligence) by joining +static code analysis nodes (Class, Method) with runtime nodes (Span, StackFrame) and +project knowledge nodes (Memory, Observation). + +All queries assume: +- CGC has indexed a PHP/Laravel repository (Method, Class, File nodes exist) +- OTEL or Xdebug plugin has written at least some runtime data +- Memory plugin has stored at least some project knowledge entries + +--- + +## 1. Execution Path for a Route + +Find every method observed at runtime for a given HTTP route, ordered by frequency. + +```cypher +MATCH (s:Span {http_route: "/api/orders"})-[:CORRELATES_TO]->(m:Method) +RETURN + m.fqn AS method, + m.class_name AS class, + count(s) AS executions, + avg(s.duration_ms) AS avg_duration_ms +ORDER BY executions DESC +LIMIT 20 +``` + +**Expected result schema**: + +| Column | Type | Description | +|--------|------|-------------| +| `method` | string | Fully-qualified method name, e.g. `App\Http\Controllers\OrderController::store` | +| `class` | string | Class name | +| `executions` | int | Number of spans that correlated to this method | +| `avg_duration_ms` | float | Average span duration in milliseconds | + +--- + +## 2. Recently Executed Methods With No Spec + +Identify code that has been observed at runtime but has no Memory/Observation linked to it. +Useful for finding undocumented hot paths. + +```cypher +MATCH (m:Method)<-[:CORRELATES_TO]-(s:Span) +WHERE NOT EXISTS { + MATCH (mem:Memory)-[:DESCRIBES]->(m) +} +RETURN + m.fqn AS method, + count(s) AS executions, + max(s.start_time_ns) AS last_seen_ns +ORDER BY executions DESC +LIMIT 20 +``` + +**Expected result schema**: + +| Column | Type | Description | +|--------|------|-------------| +| `method` | string | FQN of the method | +| `executions` | int | Total observed executions | +| `last_seen_ns` | int | Unix nanosecond timestamp of most recent span | + +--- + +## 3. Cross-Service Call Chains + +Trace spans that exit the local service boundary (CLIENT kind with `peer.service` set), +showing the full service-to-service call path. + +```cypher +MATCH path = (caller:Span)-[:CALLS_SERVICE]->(callee:Service) +MATCH (caller)-[:PART_OF]->(t:Trace) +MATCH (caller)-[:ORIGINATED_FROM]->(src:Service) +RETURN + src.name AS from_service, + callee.name AS to_service, + caller.name AS span_name, + caller.duration_ms AS duration_ms, + t.trace_id AS trace_id +ORDER BY caller.start_time_ns DESC +LIMIT 25 +``` + +**Expected result schema**: + +| Column | Type | Description | +|--------|------|-------------| +| `from_service` | string | Originating service name | +| `to_service` | string | Called downstream service name | +| `span_name` | string | Name of the CLIENT span | +| `duration_ms` | float | Duration of the outbound call | +| `trace_id` | string | Trace identifier | + +--- + +## 4. Specs Describing Recently-Active Code + +Show Memory entries that describe code observed at runtime in the last N spans. +Surfaces "well-documented hot paths". + +```cypher +MATCH (mem:Memory)-[:DESCRIBES]->(m:Method)<-[:CORRELATES_TO]-(s:Span) +RETURN + mem.name AS spec_name, + mem.entity_type AS spec_type, + m.fqn AS method, + count(s) AS executions, + collect(DISTINCT mem.content)[0..1][0] AS spec_excerpt +ORDER BY executions DESC +LIMIT 20 +``` + +**Expected result schema**: + +| Column | Type | Description | +|--------|------|-------------| +| `spec_name` | string | Memory node name | +| `spec_type` | string | Entity type (e.g. `spec`, `note`, `adr`) | +| `method` | string | FQN of the described method | +| `executions` | int | Runtime execution count | +| `spec_excerpt` | string | First 0–1 items of content for context | + +--- + +## 5. Static Code Never Observed at Runtime + +Find Method nodes with no CORRELATES_TO span and no StackFrame. Surfaces dead code +candidates or code paths never triggered in the current environment. + +```cypher +MATCH (m:Method) +WHERE NOT EXISTS { MATCH (m)<-[:CORRELATES_TO]-(:Span) } + AND NOT EXISTS { MATCH (m)<-[:RESOLVES_TO]-(:StackFrame) } + AND m.fqn IS NOT NULL +RETURN + m.fqn AS method, + m.class_name AS class, + m.file_path AS file +ORDER BY m.class_name, m.fqn +LIMIT 50 +``` + +**Expected result schema**: + +| Column | Type | Description | +|--------|------|-------------| +| `method` | string | FQN of method with no observed execution | +| `class` | string | Owning class | +| `file` | string | Source file path | + +--- + +## Running These Queries + +Via CGC CLI: + +```bash +cgc query "MATCH (m:Method)<-[:CORRELATES_TO]-(s:Span) WHERE NOT EXISTS { MATCH (mem:Memory)-[:DESCRIBES]->(m) } RETURN m.fqn, count(s) AS executions ORDER BY executions DESC LIMIT 20" +``` + +Via MCP tool (`otel_cross_layer_query`): + +```json +{ + "tool": "otel_cross_layer_query", + "arguments": {"query_type": "unspecced_running_code"} +} +``` + +Via Neo4j Browser: connect to `bolt://localhost:7687` and paste any query above. diff --git a/docs/plugins/examples/send_test_span.py b/docs/plugins/examples/send_test_span.py new file mode 100644 index 00000000..c79fe412 --- /dev/null +++ b/docs/plugins/examples/send_test_span.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +""" +Send a synthetic OTLP span to the CGC OTEL Collector for manual testing. + +Usage: + pip install opentelemetry-sdk opentelemetry-exporter-otlp-proto-grpc + python docs/plugins/examples/send_test_span.py + + # Custom endpoint: + OTEL_ENDPOINT=localhost:4317 python docs/plugins/examples/send_test_span.py + +Verifying results in Neo4j Browser (http://localhost:7474): + MATCH (s:Span) RETURN s.name, s.http_route, s.duration_ms LIMIT 10 + MATCH (s:Service) RETURN s.name +""" +import os +import time + +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import BatchSpanProcessor +from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter +from opentelemetry.sdk.resources import Resource + +ENDPOINT = os.environ.get("OTEL_ENDPOINT", "localhost:4317") +SERVICE_NAME = os.environ.get("OTEL_SERVICE_NAME", "cgc-test-service") + + +def send_test_spans(): + resource = Resource.create({"service.name": SERVICE_NAME}) + provider = TracerProvider(resource=resource) + + exporter = OTLPSpanExporter(endpoint=ENDPOINT, insecure=True) + provider.add_span_processor(BatchSpanProcessor(exporter)) + trace.set_tracer_provider(provider) + + tracer = trace.get_tracer("cgc.manual.test") + + print(f"Sending test spans to {ENDPOINT} (service: {SERVICE_NAME})...") + + # Simulate an HTTP request trace with a DB child span + with tracer.start_as_current_span("GET /api/orders") as root_span: + root_span.set_attribute("http.method", "GET") + root_span.set_attribute("http.route", "/api/orders") + root_span.set_attribute("http.status_code", 200) + root_span.set_attribute("code.namespace", "App\\Http\\Controllers") + root_span.set_attribute("code.function", "OrderController::index") + + time.sleep(0.01) # simulate work + + with tracer.start_as_current_span("DB: SELECT orders") as child_span: + child_span.set_attribute("db.system", "mysql") + child_span.set_attribute("db.statement", "SELECT * FROM orders LIMIT 10") + child_span.set_attribute("peer.service", "mysql") + time.sleep(0.005) + + # Simulate a second, different route + with tracer.start_as_current_span("POST /api/orders") as span2: + span2.set_attribute("http.method", "POST") + span2.set_attribute("http.route", "/api/orders") + span2.set_attribute("http.status_code", 201) + span2.set_attribute("code.namespace", "App\\Http\\Controllers") + span2.set_attribute("code.function", "OrderController::store") + time.sleep(0.02) + + # Flush + provider.force_flush() + print("Done. Check Neo4j: MATCH (s:Span) RETURN s.name, s.http_route LIMIT 10") + + +if __name__ == "__main__": + send_test_spans() diff --git a/docs/plugins/manual-testing.md b/docs/plugins/manual-testing.md new file mode 100644 index 00000000..ca061a06 --- /dev/null +++ b/docs/plugins/manual-testing.md @@ -0,0 +1,268 @@ +# Manual Testing Guide — CGC Plugin Stack + +Step-by-step instructions for spinning up the full plugin stack locally and verifying +each plugin works end-to-end. + +--- + +## Prerequisites + +- Docker + Docker Compose v2 (`docker compose version`) +- Python 3.10+ and pip (for CLI testing without Docker) +- `grpcurl` (optional, for OTEL gRPC smoke test — `brew install grpcurl`) +- A PHP application with OpenTelemetry SDK installed (for OTEL live test — optional) + +--- + +## Option A: Docker Stack (Recommended) + +### 1. Start the stack + +```bash +cp .env.example .env +# .env defaults work for local testing — change NEO4J_PASSWORD for anything non-local + +docker compose -f docker-compose.plugin-stack.yml up -d --build +``` + +Watch startup (Neo4j takes ~30s): +```bash +docker compose -f docker-compose.plugin-stack.yml logs -f +``` + +### 2. Verify all services are healthy + +```bash +docker compose -f docker-compose.plugin-stack.yml ps +``` + +Expected: all services show `healthy` or `running`. + +| Service | Port | Check | +|---|---|---| +| neo4j | 7474, 7687 | http://localhost:7474 → Neo4j Browser | +| cgc-otel-processor | 5317 | `docker logs cgc-otel-processor` → no errors | +| otel-collector | 4317, 4318 | `docker logs cgc-otel-collector` → "Everything is ready" | +| cgc-memory | 8766 | `docker logs cgc-memory` → no errors | + +### 3. Verify graph schema initialized + +Open http://localhost:7474, login (neo4j / codegraph123), run: + +```cypher +SHOW CONSTRAINTS +``` + +Expected: `service_name`, `trace_id`, `span_id`, `frame_id` constraints present. + +```cypher +SHOW INDEXES +``` + +Expected: `memory_search`, `observation_search` FULLTEXT indexes present. + +--- + +## Option B: Python (No Docker) + +Install everything editable in a venv: + +```bash +python -m venv .venv && source .venv/bin/activate +pip install -e . +pip install -e plugins/cgc-plugin-stub +pip install -e plugins/cgc-plugin-otel +pip install -e plugins/cgc-plugin-xdebug +pip install -e plugins/cgc-plugin-memory +``` + +Verify plugin discovery: +```bash +PYTHONPATH=src cgc plugin list +# Should show all four plugins as "loaded" + +PYTHONPATH=src cgc --help +# Should show: stub, otel, xdebug, memory command groups +``` + +--- + +## Testing Each Plugin + +### Stub Plugin (smoke test — no DB needed) + +```bash +# CLI +PYTHONPATH=src cgc stub hello +# Expected: "Hello from stub plugin" + +PYTHONPATH=src cgc stub hello --name "Alice" +# Expected: "Hello Alice from stub plugin" +``` + +Via pytest (no install needed for mocked path): +```bash +PYTHONPATH=src pytest tests/unit/plugin/test_plugin_registry.py -v +``` + +--- + +### Memory Plugin + +**Requires**: Neo4j running at `bolt://localhost:7687` + +```bash +# Store a spec +PYTHONPATH=src cgc memory store \ + --type spec \ + --name "OrderController spec" \ + --content "Handles order creation and payment transitions" + +# Search +PYTHONPATH=src cgc memory search --query "order" + +# List undocumented classes +PYTHONPATH=src cgc memory undocumented + +# Status +PYTHONPATH=src cgc memory status +``` + +Verify in Neo4j Browser: +```cypher +MATCH (m:Memory) RETURN m.name, m.entity_type, m.content LIMIT 10 +``` + +--- + +### OTEL Plugin + +**Requires**: Neo4j + `cgc-otel-processor` + `otel-collector` running. + +#### Send a synthetic span + +Using `grpcurl` (easiest): +```bash +# Check collector is accepting connections +grpcurl -plaintext localhost:4317 list +# Expected: opentelemetry.proto.collector.trace.v1.TraceService +``` + +Using a Python script: +```bash +python docs/plugins/examples/send_test_span.py +# See docs/plugins/examples/ for this script +``` + +#### Configure a PHP/Laravel app + +Add to your app's `.env`: +```ini +OTEL_PHP_AUTOLOAD_ENABLED=true +OTEL_SERVICE_NAME=my-laravel-app +OTEL_TRACES_EXPORTER=otlp +OTEL_EXPORTER_OTLP_PROTOCOL=grpc +OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 +``` + +Send a request to your app, then query: +```bash +PYTHONPATH=src cgc otel query-spans --route "/api/orders" --limit 5 +PYTHONPATH=src cgc otel list-services +``` + +Verify in Neo4j Browser: +```cypher +MATCH (s:Service) RETURN s.name +MATCH (sp:Span) RETURN sp.name, sp.duration_ms, sp.http_route LIMIT 10 +MATCH (sp:Span)-[:CORRELATES_TO]->(m:Method) RETURN sp.name, m.fqn LIMIT 10 +``` + +--- + +### Xdebug Plugin + +**Requires**: Neo4j + PHP with Xdebug installed. + +Start the listener (Docker): +```bash +docker compose -f docker-compose.plugin-stack.yml -f docker-compose.dev.yml up -d xdebug-listener +docker logs cgc-xdebug-listener -f +# Expected: "DBGp server listening on 0.0.0.0:9003" +``` + +Start the listener (Python): +```bash +CGC_PLUGIN_XDEBUG_ENABLED=true PYTHONPATH=src cgc xdebug start +``` + +Configure PHP (`php.ini` or `.env`): +```ini +xdebug.mode=debug +xdebug.client_host=localhost +xdebug.client_port=9003 +xdebug.start_with_request=trigger +``` + +Trigger a trace by setting the `XDEBUG_TRIGGER` cookie in your browser, then: +```bash +PYTHONPATH=src cgc xdebug list-chains --limit 10 +PYTHONPATH=src cgc xdebug status +``` + +Verify in Neo4j Browser: +```cypher +MATCH (sf:StackFrame) RETURN sf.class_name, sf.method_name, sf.observation_count LIMIT 20 +MATCH (sf:StackFrame)-[:CALLED_BY]->(parent:StackFrame) RETURN sf.method_name, parent.method_name LIMIT 10 +MATCH (sf:StackFrame)-[:RESOLVES_TO]->(m:Method) RETURN sf.method_name, m.fqn LIMIT 10 +``` + +--- + +## Cross-Layer Validation + +After running all plugins with real data, validate the cross-layer queries: + +```bash +# Methods running with no spec +PYTHONPATH=src cgc query " +MATCH (m:Method)<-[:CORRELATES_TO]-(s:Span) +WHERE NOT EXISTS { MATCH (mem:Memory)-[:DESCRIBES]->(m) } +RETURN m.fqn, count(s) AS executions +ORDER BY executions DESC LIMIT 10 +" + +# Static code never observed at runtime +PYTHONPATH=src cgc query " +MATCH (m:Method) +WHERE NOT EXISTS { MATCH (m)<-[:CORRELATES_TO]-(:Span) } + AND NOT EXISTS { MATCH (m)<-[:RESOLVES_TO]-(:StackFrame) } +RETURN m.fqn, m.class_name LIMIT 10 +" +``` + +See `docs/plugins/cross-layer-queries.md` for all 5 canonical queries. + +--- + +## Teardown + +```bash +# Stop all services +docker compose -f docker-compose.plugin-stack.yml down + +# Remove volumes (clears Neo4j data) +docker compose -f docker-compose.plugin-stack.yml down -v +``` + +--- + +## Troubleshooting + +| Symptom | Likely cause | Fix | +|---|---|---| +| `service_healthy` wait times out | Neo4j slow to start | Increase `start_period` in healthcheck or wait longer | +| `cgc plugin list` shows plugin as failed | Plugin not installed | `pip install -e plugins/cgc-plugin-` | +| Spans sent but no graph nodes | Filter routes dropping them | Check `OTEL_FILTER_ROUTES`; default drops `/health` etc. | +| Xdebug not connecting | Wrong `client_host` | Use Docker host IP, not `localhost`, when PHP is in Docker | +| Memory search returns nothing | FULLTEXT index not created | Run `config/neo4j/init.cypher` manually in Neo4j Browser | diff --git a/k8s/cgc-plugin-memory/deployment.yaml b/k8s/cgc-plugin-memory/deployment.yaml new file mode 100644 index 00000000..387fda78 --- /dev/null +++ b/k8s/cgc-plugin-memory/deployment.yaml @@ -0,0 +1,67 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cgc-memory + labels: + app: cgc-memory + app.kubernetes.io/part-of: codegraphcontext +spec: + replicas: 1 + selector: + matchLabels: + app: cgc-memory + template: + metadata: + labels: + app: cgc-memory + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: cgc-memory + image: ghcr.io/codegraphcontext/cgc-plugin-memory:latest + imagePullPolicy: IfNotPresent + ports: + - name: mcp + containerPort: 8766 + protocol: TCP + env: + - name: NEO4J_URI + valueFrom: + configMapKeyRef: + name: cgc-config + key: NEO4J_URI + - name: NEO4J_USERNAME + valueFrom: + configMapKeyRef: + name: cgc-config + key: NEO4J_USERNAME + - name: NEO4J_PASSWORD + valueFrom: + secretKeyRef: + name: cgc-secrets + key: NEO4J_PASSWORD + - name: NEO4J_DATABASE + value: "neo4j" + - name: CGC_MEMORY_PORT + value: "8766" + - name: LOG_LEVEL + value: "INFO" + readinessProbe: + exec: + command: ["python", "-c", "import cgc_plugin_memory; print('ok')"] + initialDelaySeconds: 10 + periodSeconds: 15 + livenessProbe: + exec: + command: ["python", "-c", "import cgc_plugin_memory; print('ok')"] + initialDelaySeconds: 20 + periodSeconds: 30 + resources: + requests: + cpu: "50m" + memory: "64Mi" + limits: + cpu: "250m" + memory: "256Mi" diff --git a/k8s/cgc-plugin-memory/service.yaml b/k8s/cgc-plugin-memory/service.yaml new file mode 100644 index 00000000..88cba379 --- /dev/null +++ b/k8s/cgc-plugin-memory/service.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: cgc-memory + labels: + app: cgc-memory + app.kubernetes.io/part-of: codegraphcontext +spec: + type: ClusterIP + selector: + app: cgc-memory + ports: + - name: mcp + port: 8766 + targetPort: mcp + protocol: TCP diff --git a/k8s/cgc-plugin-otel/deployment.yaml b/k8s/cgc-plugin-otel/deployment.yaml new file mode 100644 index 00000000..bff4f2c8 --- /dev/null +++ b/k8s/cgc-plugin-otel/deployment.yaml @@ -0,0 +1,66 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cgc-otel-processor + labels: + app: cgc-otel-processor + app.kubernetes.io/part-of: codegraphcontext +spec: + replicas: 1 + selector: + matchLabels: + app: cgc-otel-processor + template: + metadata: + labels: + app: cgc-otel-processor + spec: + securityContext: + runAsNonRoot: true + runAsUser: 1000 + containers: + - name: cgc-otel-processor + image: ghcr.io/codegraphcontext/cgc-plugin-otel:latest + imagePullPolicy: IfNotPresent + ports: + - name: grpc-receiver + containerPort: 5317 + protocol: TCP + env: + - name: NEO4J_URI + valueFrom: + configMapKeyRef: + name: cgc-config + key: NEO4J_URI + - name: NEO4J_USERNAME + valueFrom: + configMapKeyRef: + name: cgc-config + key: NEO4J_USERNAME + - name: NEO4J_PASSWORD + valueFrom: + secretKeyRef: + name: cgc-secrets + key: NEO4J_PASSWORD + - name: OTEL_RECEIVER_PORT + value: "5317" + - name: LOG_LEVEL + value: "INFO" + readinessProbe: + exec: + command: ["python", "-c", "import grpc; print('ok')"] + initialDelaySeconds: 10 + periodSeconds: 15 + failureThreshold: 3 + livenessProbe: + exec: + command: ["python", "-c", "import grpc; print('ok')"] + initialDelaySeconds: 20 + periodSeconds: 30 + resources: + requests: + cpu: "100m" + memory: "128Mi" + limits: + cpu: "500m" + memory: "512Mi" diff --git a/k8s/cgc-plugin-otel/service.yaml b/k8s/cgc-plugin-otel/service.yaml new file mode 100644 index 00000000..4bf9d1bf --- /dev/null +++ b/k8s/cgc-plugin-otel/service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: cgc-otel-processor + labels: + app: cgc-otel-processor + app.kubernetes.io/part-of: codegraphcontext +spec: + type: ClusterIP + selector: + app: cgc-otel-processor + ports: + - name: grpc-receiver + port: 5317 + targetPort: grpc-receiver + protocol: TCP + - name: http-otlp + port: 4318 + targetPort: 4318 + protocol: TCP diff --git a/plugins/cgc-plugin-memory/Dockerfile b/plugins/cgc-plugin-memory/Dockerfile new file mode 100644 index 00000000..9d317feb --- /dev/null +++ b/plugins/cgc-plugin-memory/Dockerfile @@ -0,0 +1,20 @@ +FROM python:3.12-slim + +RUN groupadd -r cgc && useradd -r -g cgc cgc + +WORKDIR /app + +COPY pyproject.toml README.md ./ +COPY src/ ./src/ + +RUN pip install --no-cache-dir -e . && \ + pip install --no-cache-dir "typer[all]>=0.9.0" "neo4j>=5.15.0" || true + +USER cgc + +EXPOSE 8766 + +HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \ + CMD python -c "import cgc_plugin_memory; print('ok')" || exit 1 + +CMD ["python", "-m", "cgc_plugin_memory.server"] diff --git a/plugins/cgc-plugin-memory/pyproject.toml b/plugins/cgc-plugin-memory/pyproject.toml new file mode 100644 index 00000000..c31b6d59 --- /dev/null +++ b/plugins/cgc-plugin-memory/pyproject.toml @@ -0,0 +1,35 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "cgc-plugin-memory" +version = "0.1.0" +description = "Project knowledge memory plugin for CodeGraphContext" +readme = "README.md" +requires-python = ">=3.10" +authors = [ + { name = "CodeGraphContext Contributors" } +] +dependencies = [ + "codegraphcontext>=0.3.0", + "typer[all]>=0.9.0", + "neo4j>=5.15.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.4.0", + "pytest-asyncio>=0.21.0", + "pytest-mock>=3.11.0", +] + +[project.entry-points."cgc_cli_plugins"] +memory = "cgc_plugin_memory.cli:get_plugin_commands" + +[project.entry-points."cgc_mcp_plugins"] +memory = "cgc_plugin_memory.mcp_tools:get_mcp_tools" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["cgc_plugin_memory*"] diff --git a/plugins/cgc-plugin-memory/src/cgc_plugin_memory/__init__.py b/plugins/cgc-plugin-memory/src/cgc_plugin_memory/__init__.py new file mode 100644 index 00000000..bd9db99a --- /dev/null +++ b/plugins/cgc-plugin-memory/src/cgc_plugin_memory/__init__.py @@ -0,0 +1,12 @@ +"""Memory plugin for CodeGraphContext — stores and searches project knowledge in the graph.""" + +PLUGIN_METADATA = { + "name": "cgc-plugin-memory", + "version": "0.1.0", + "cgc_version_constraint": ">=0.1.0", + "description": ( + "Exposes MCP tools and CLI commands to store, search, and link knowledge " + "entities (specs, decisions, notes) in the Neo4j graph, enabling cross-layer " + "queries like 'which code has no spec?'." + ), +} diff --git a/plugins/cgc-plugin-memory/src/cgc_plugin_memory/cli.py b/plugins/cgc-plugin-memory/src/cgc_plugin_memory/cli.py new file mode 100644 index 00000000..8f114ddc --- /dev/null +++ b/plugins/cgc-plugin-memory/src/cgc_plugin_memory/cli.py @@ -0,0 +1,86 @@ +"""CLI command group contributed by the Memory plugin.""" +from __future__ import annotations + +import typer +from typing import Optional + +memory_app = typer.Typer(name="memory", help="Project knowledge memory commands.") + + +@memory_app.command("store") +def store( + entity_type: str = typer.Option(..., "--type", help="Knowledge type (spec, decision, note, …)"), + name: str = typer.Option(..., "--name", help="Short descriptive name"), + content: str = typer.Option(..., "--content", help="Full content / body text"), + links_to: Optional[str] = typer.Option(None, "--links-to", help="FQN of code node to link via DESCRIBES"), +): + """Store a knowledge entity in the graph.""" + try: + from codegraphcontext.core import get_database_manager + db = get_database_manager() + except Exception as e: + typer.echo(f"Database unavailable: {e}", err=True) + raise typer.Exit(1) + + from cgc_plugin_memory.mcp_tools import _make_store_handler + result = _make_store_handler(db)(entity_type=entity_type, name=name, content=content, links_to=links_to) + typer.echo(f"Stored memory {result['memory_id']}") + + +@memory_app.command("search") +def search( + query: str = typer.Argument(..., help="Search terms"), + limit: int = typer.Option(10, "--limit"), +): + """Full-text search across stored memories.""" + try: + from codegraphcontext.core import get_database_manager + db = get_database_manager() + except Exception as e: + typer.echo(f"Database unavailable: {e}", err=True) + raise typer.Exit(1) + + from cgc_plugin_memory.mcp_tools import _make_search_handler + result = _make_search_handler(db)(query=query, limit=limit) + if not result["results"]: + typer.echo("No results found.") + return + for row in result["results"]: + typer.echo(f"[{row.get('entity_type')}] {row.get('name')} (score: {row.get('score', '?'):.3f})") + typer.echo(f" {str(row.get('content',''))[:120]}") + + +@memory_app.command("undocumented") +def undocumented( + node_type: str = typer.Option("Class", "--type", help="Class or Method"), + limit: int = typer.Option(20, "--limit"), +): + """List code nodes that have no linked Memory (no documentation/spec).""" + try: + from codegraphcontext.core import get_database_manager + db = get_database_manager() + except Exception as e: + typer.echo(f"Database unavailable: {e}", err=True) + raise typer.Exit(1) + + from cgc_plugin_memory.mcp_tools import _make_undocumented_handler + result = _make_undocumented_handler(db)(node_type=node_type, limit=limit) + if not result["nodes"]: + typer.echo(f"All {node_type} nodes are documented.") + return + typer.echo(f"Undocumented {node_type} nodes:") + for row in result["nodes"]: + typer.echo(f" {row.get('fqn')}") + + +@memory_app.command("status") +def status(): + """Show Memory plugin status.""" + typer.echo("Memory plugin is active.") + typer.echo("Use 'cgc memory store' to add knowledge entities.") + typer.echo("Use 'cgc memory undocumented' to find unspecced code.") + + +def get_plugin_commands() -> tuple[str, typer.Typer]: + """Entry point: return (command_name, typer_app).""" + return ("memory", memory_app) diff --git a/plugins/cgc-plugin-memory/src/cgc_plugin_memory/mcp_tools.py b/plugins/cgc-plugin-memory/src/cgc_plugin_memory/mcp_tools.py new file mode 100644 index 00000000..40196dc0 --- /dev/null +++ b/plugins/cgc-plugin-memory/src/cgc_plugin_memory/mcp_tools.py @@ -0,0 +1,213 @@ +"""MCP tools contributed by the Memory plugin.""" +from __future__ import annotations + +import uuid +from typing import Any + +_TOOLS: dict[str, dict] = { + "memory_store": { + "name": "memory_store", + "description": ( + "Store a knowledge entity (spec, decision, note, etc.) in the graph. " + "Optionally link it to a code node by its fully-qualified name." + ), + "inputSchema": { + "type": "object", + "properties": { + "entity_type": { + "type": "string", + "description": "Type of knowledge (spec, decision, note, requirement, …)", + }, + "name": {"type": "string", "description": "Short descriptive name"}, + "content": {"type": "string", "description": "Full content / body text"}, + "links_to": { + "type": "string", + "description": "FQN of a Class or Method node to link this memory to via DESCRIBES", + }, + }, + "required": ["entity_type", "name", "content"], + }, + }, + "memory_search": { + "name": "memory_search", + "description": "Full-text search across stored Memory nodes.", + "inputSchema": { + "type": "object", + "properties": { + "query": {"type": "string", "description": "Search terms"}, + "limit": {"type": "integer", "default": 10}, + }, + "required": ["query"], + }, + }, + "memory_undocumented": { + "name": "memory_undocumented", + "description": ( + "Return Class or Method nodes that have no linked Memory node (no DESCRIBES relationship). " + "Helps identify code that lacks specs or documentation." + ), + "inputSchema": { + "type": "object", + "properties": { + "node_type": { + "type": "string", + "enum": ["Class", "Method"], + "default": "Class", + "description": "Type of code node to check", + }, + "limit": {"type": "integer", "default": 20}, + }, + "required": [], + }, + }, + "memory_link": { + "name": "memory_link", + "description": "Create a DESCRIBES relationship between a Memory node and a code node.", + "inputSchema": { + "type": "object", + "properties": { + "memory_id": {"type": "string", "description": "The memory_id of the Memory node"}, + "node_fqn": { + "type": "string", + "description": "Fully-qualified name of the Class or Method node", + }, + "node_type": { + "type": "string", + "enum": ["Class", "Method"], + "description": "Label of the target node", + }, + }, + "required": ["memory_id", "node_fqn", "node_type"], + }, + }, +} + +# --------------------------------------------------------------------------- +# Cypher +# --------------------------------------------------------------------------- + +_MERGE_MEMORY = """ +MERGE (m:Memory {memory_id: $memory_id}) +ON CREATE SET + m.entity_type = $entity_type, + m.name = $name, + m.content = $content, + m.created_at = datetime() +ON MATCH SET + m.content = $content, + m.updated_at = datetime() +""" + +_MERGE_DESCRIBES = """ +MATCH (m:Memory {memory_id: $memory_id}) +MATCH (n {fqn: $node_fqn}) +WHERE $node_type IN labels(n) +MERGE (m)-[:DESCRIBES]->(n) +""" + +_FULLTEXT_SEARCH = """ +CALL db.index.fulltext.queryNodes('memory_search', $query) +YIELD node AS m, score +RETURN m.memory_id AS memory_id, m.name AS name, m.entity_type AS entity_type, + m.content AS content, score +ORDER BY score DESC LIMIT $limit +""" + +_UNDOCUMENTED = "MATCH (n:{node_type}) WHERE NOT EXISTS {{ MATCH (m:Memory)-[:DESCRIBES]->(n) }} RETURN n.fqn AS fqn, labels(n) AS type ORDER BY n.fqn LIMIT $limit" + +_LINK_DESCRIBES = """ +MATCH (m:Memory {memory_id: $memory_id}) +MATCH (n) +WHERE n.fqn = $node_fqn AND $node_type IN labels(n) +MERGE (m)-[:DESCRIBES]->(n) +""" + + +# --------------------------------------------------------------------------- +# Handler factories +# --------------------------------------------------------------------------- + +def _make_store_handler(db_manager: Any): + def handle( + entity_type: str, + name: str, + content: str, + links_to: str | None = None, + **_: Any, + ) -> dict: + memory_id = str(uuid.uuid4()) + driver = db_manager.get_driver() + with driver.session() as session: + session.run( + _MERGE_MEMORY, + memory_id=memory_id, + entity_type=entity_type, + name=name, + content=content, + ) + if links_to: + # Attempt to link to Class first, then Method + for node_type in ("Class", "Method"): + session.run( + _MERGE_DESCRIBES, + memory_id=memory_id, + node_fqn=links_to, + node_type=node_type, + ) + return {"memory_id": memory_id, "status": "stored"} + return handle + + +def _make_search_handler(db_manager: Any): + def handle(query: str, limit: int = 10, **_: Any) -> dict: + driver = db_manager.get_driver() + with driver.session() as session: + rows = session.run(_FULLTEXT_SEARCH, query=query, limit=limit).data() + return {"results": rows} + return handle + + +def _make_undocumented_handler(db_manager: Any): + def handle(node_type: str = "Class", limit: int = 20, **_: Any) -> dict: + # Node labels cannot be parameterized in Cypher — interpolate safely + # (node_type is validated against enum in the tool schema) + safe_type = node_type if node_type in ("Class", "Method") else "Class" + cypher = _UNDOCUMENTED.format(node_type=safe_type) + driver = db_manager.get_driver() + with driver.session() as session: + rows = session.run(cypher, limit=limit).data() + return {"nodes": rows} + return handle + + +def _make_link_handler(db_manager: Any): + def handle(memory_id: str, node_fqn: str, node_type: str = "Class", **_: Any) -> dict: + driver = db_manager.get_driver() + with driver.session() as session: + session.run(_LINK_DESCRIBES, memory_id=memory_id, node_fqn=node_fqn, node_type=node_type) + return {"status": "linked"} + return handle + + +# --------------------------------------------------------------------------- +# Entry points +# --------------------------------------------------------------------------- + +def get_mcp_tools(server_context: dict | None = None) -> dict[str, dict]: + """Entry point: return tool_name → ToolDefinition mapping.""" + return _TOOLS + + +def get_mcp_handlers(server_context: dict | None = None) -> dict[str, Any]: + """Entry point: return tool_name → callable mapping.""" + if server_context is None: + server_context = {} + db_manager = server_context.get("db_manager") + if db_manager is None: + return {} + return { + "memory_store": _make_store_handler(db_manager), + "memory_search": _make_search_handler(db_manager), + "memory_undocumented": _make_undocumented_handler(db_manager), + "memory_link": _make_link_handler(db_manager), + } diff --git a/plugins/cgc-plugin-otel/Dockerfile b/plugins/cgc-plugin-otel/Dockerfile new file mode 100644 index 00000000..e72d8549 --- /dev/null +++ b/plugins/cgc-plugin-otel/Dockerfile @@ -0,0 +1,22 @@ +FROM python:3.12-slim + +# Security: run as non-root +RUN groupadd -r cgc && useradd -r -g cgc cgc + +WORKDIR /app + +COPY pyproject.toml README.md ./ +COPY src/ ./src/ + +RUN pip install --no-cache-dir -e ".[dev]" 2>/dev/null || pip install --no-cache-dir -e . && \ + pip install --no-cache-dir grpcio>=1.57.0 "opentelemetry-proto>=0.43b0" "opentelemetry-sdk>=1.20.0" \ + "typer[all]>=0.9.0" "neo4j>=5.15.0" || true + +USER cgc + +EXPOSE 5317 + +HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \ + CMD python -c "import grpc; print('ok')" || exit 1 + +CMD ["python", "-m", "cgc_plugin_otel.receiver"] diff --git a/plugins/cgc-plugin-otel/pyproject.toml b/plugins/cgc-plugin-otel/pyproject.toml new file mode 100644 index 00000000..ac3b07f6 --- /dev/null +++ b/plugins/cgc-plugin-otel/pyproject.toml @@ -0,0 +1,40 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "cgc-plugin-otel" +version = "0.1.0" +description = "OpenTelemetry span processor plugin for CodeGraphContext" +readme = "README.md" +requires-python = ">=3.10" +authors = [ + { name = "CodeGraphContext Contributors" } +] +dependencies = [ + "codegraphcontext>=0.3.0", + "typer[all]>=0.9.0", + "neo4j>=5.15.0", + "grpcio>=1.57.0", + "grpcio-tools>=1.57.0", + "opentelemetry-proto>=0.43b0", + "opentelemetry-sdk>=1.20.0", + "opentelemetry-api>=1.20.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.4.0", + "pytest-asyncio>=0.21.0", + "pytest-mock>=3.11.0", +] + +[project.entry-points."cgc_cli_plugins"] +otel = "cgc_plugin_otel.cli:get_plugin_commands" + +[project.entry-points."cgc_mcp_plugins"] +otel = "cgc_plugin_otel.mcp_tools:get_mcp_tools" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["cgc_plugin_otel*"] diff --git a/plugins/cgc-plugin-otel/src/cgc_plugin_otel/__init__.py b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/__init__.py new file mode 100644 index 00000000..66bbe9e6 --- /dev/null +++ b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/__init__.py @@ -0,0 +1,11 @@ +"""OTEL plugin for CodeGraphContext — receives OpenTelemetry spans and writes them to the graph.""" + +PLUGIN_METADATA = { + "name": "cgc-plugin-otel", + "version": "0.1.0", + "cgc_version_constraint": ">=0.1.0", + "description": ( + "Receives OpenTelemetry traces via gRPC, writes Service/Trace/Span nodes to the " + "code graph, and correlates runtime spans to static Method nodes." + ), +} diff --git a/plugins/cgc-plugin-otel/src/cgc_plugin_otel/cli.py b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/cli.py new file mode 100644 index 00000000..16ae1cbe --- /dev/null +++ b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/cli.py @@ -0,0 +1,83 @@ +"""CLI command group contributed by the OTEL plugin.""" +from __future__ import annotations + +import os +import typer +from typing import Optional + +otel_app = typer.Typer(name="otel", help="OpenTelemetry span commands.") + + +@otel_app.command("query-spans") +def query_spans( + route: Optional[str] = typer.Option(None, "--route", help="Filter by HTTP route"), + service: Optional[str] = typer.Option(None, "--service", help="Filter by service name"), + limit: int = typer.Option(20, "--limit", help="Maximum results"), +): + """Query spans stored in the graph, optionally filtered by route or service.""" + try: + from codegraphcontext.core import get_database_manager + db = get_database_manager() + except Exception as e: + typer.echo(f"Database unavailable: {e}", err=True) + raise typer.Exit(1) + + where_clauses = [] + params: dict = {"limit": limit} + if route: + where_clauses.append("sp.http_route = $route") + params["route"] = route + if service: + where_clauses.append("sp.service_name = $service") + params["service"] = service + + where = ("WHERE " + " AND ".join(where_clauses)) if where_clauses else "" + cypher = f"MATCH (sp:Span) {where} RETURN sp.span_id, sp.name, sp.service_name, sp.duration_ms ORDER BY sp.start_time_ns DESC LIMIT $limit" + + try: + driver = db.get_driver() + with driver.session() as session: + result = session.run(cypher, **params) + rows = result.data() + except Exception as e: + typer.echo(f"Query failed: {e}", err=True) + raise typer.Exit(1) + + if not rows: + typer.echo("No spans found.") + return + for row in rows: + typer.echo(f"[{row.get('sp.service_name')}] {row.get('sp.name')} — {row.get('sp.duration_ms', '?')}ms id={row.get('sp.span_id')}") + + +@otel_app.command("list-services") +def list_services(): + """List all services observed in the span graph.""" + try: + from codegraphcontext.core import get_database_manager + db = get_database_manager() + driver = db.get_driver() + with driver.session() as session: + rows = session.run("MATCH (s:Service) RETURN s.name ORDER BY s.name").data() + except Exception as e: + typer.echo(f"Error: {e}", err=True) + raise typer.Exit(1) + + if not rows: + typer.echo("No services found.") + return + for row in rows: + typer.echo(row["s.name"]) + + +@otel_app.command("status") +def status(): + """Show whether the OTEL receiver process is configured.""" + port = os.environ.get("OTEL_RECEIVER_PORT", "5317") + typer.echo(f"OTEL receiver port: {port}") + typer.echo("Run 'python -m cgc_plugin_otel.receiver' to start the gRPC receiver.") + + +def get_plugin_commands() -> tuple[str, typer.Typer]: + """Entry point: return (command_name, typer_app).""" + return ("otel", otel_app) diff --git a/plugins/cgc-plugin-otel/src/cgc_plugin_otel/mcp_tools.py b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/mcp_tools.py new file mode 100644 index 00000000..7c089c5f --- /dev/null +++ b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/mcp_tools.py @@ -0,0 +1,128 @@ +"""MCP tools contributed by the OTEL plugin.""" +from __future__ import annotations + +from typing import Any + +_TOOLS: dict[str, dict] = { + "otel_query_spans": { + "name": "otel_query_spans", + "description": ( + "Query OpenTelemetry spans stored in the graph. " + "Filter by HTTP route and/or service name." + ), + "inputSchema": { + "type": "object", + "properties": { + "http_route": {"type": "string", "description": "Filter by HTTP route (e.g. /api/orders)"}, + "service": {"type": "string", "description": "Filter by service name"}, + "limit": {"type": "integer", "description": "Max results", "default": 20}, + }, + "required": [], + }, + }, + "otel_list_services": { + "name": "otel_list_services", + "description": "List all services observed in the runtime span graph.", + "inputSchema": {"type": "object", "properties": {}, "required": []}, + }, + "otel_cross_layer_query": { + "name": "otel_cross_layer_query", + "description": ( + "Run a pre-built cross-layer query combining static code structure with runtime spans. " + "query_type options: unspecced_running_code | cross_service_calls | recent_executions" + ), + "inputSchema": { + "type": "object", + "properties": { + "query_type": { + "type": "string", + "enum": ["unspecced_running_code", "cross_service_calls", "recent_executions"], + "description": "The cross-layer query to run", + }, + "limit": {"type": "integer", "default": 20}, + }, + "required": ["query_type"], + }, + }, +} + +_CROSS_LAYER_QUERIES: dict[str, str] = { + "unspecced_running_code": ( + "MATCH (sp:Span)-[:CORRELATES_TO]->(m:Method) " + "WHERE NOT EXISTS { MATCH (m)<-[:DESCRIBES]-(:Memory) } " + "RETURN m.fqn AS fqn, count(sp) AS run_count " + "ORDER BY run_count DESC LIMIT $limit" + ), + "cross_service_calls": ( + "MATCH (sp:Span)-[:CALLS_SERVICE]->(svc:Service) " + "RETURN sp.service_name AS caller, svc.name AS callee, sp.http_route AS route, count(*) AS calls " + "ORDER BY calls DESC LIMIT $limit" + ), + "recent_executions": ( + "MATCH (sp:Span)-[:CORRELATES_TO]->(m:Method) " + "RETURN sp.name AS span, m.fqn AS fqn, sp.duration_ms AS duration_ms " + "ORDER BY sp.start_time_ns DESC LIMIT $limit" + ), +} + + +def _make_query_spans_handler(db_manager: Any): + def handle(http_route: str | None = None, service: str | None = None, limit: int = 20) -> dict: + where_clauses = [] + params: dict = {"limit": limit} + if http_route: + where_clauses.append("sp.http_route = $http_route") + params["http_route"] = http_route + if service: + where_clauses.append("sp.service_name = $service") + params["service"] = service + where = ("WHERE " + " AND ".join(where_clauses)) if where_clauses else "" + cypher = ( + f"MATCH (sp:Span) {where} " + "RETURN sp.span_id AS span_id, sp.name AS name, sp.service_name AS service, " + "sp.duration_ms AS duration_ms, sp.http_route AS http_route " + "ORDER BY sp.start_time_ns DESC LIMIT $limit" + ) + driver = db_manager.get_driver() + with driver.session() as session: + return {"spans": session.run(cypher, **params).data()} + return handle + + +def _make_list_services_handler(db_manager: Any): + def handle(**_kwargs: Any) -> dict: + driver = db_manager.get_driver() + with driver.session() as session: + rows = session.run("MATCH (s:Service) RETURN s.name AS name ORDER BY s.name").data() + return {"services": [r["name"] for r in rows]} + return handle + + +def _make_cross_layer_handler(db_manager: Any): + def handle(query_type: str, limit: int = 20, **_kwargs: Any) -> dict: + cypher = _CROSS_LAYER_QUERIES.get(query_type) + if cypher is None: + return {"error": f"Unknown query_type '{query_type}'"} + driver = db_manager.get_driver() + with driver.session() as session: + return {"results": session.run(cypher, limit=limit).data()} + return handle + + +def get_mcp_tools(server_context: dict | None = None) -> dict[str, dict]: + """Entry point: return tool_name → ToolDefinition mapping.""" + return _TOOLS + + +def get_mcp_handlers(server_context: dict | None = None) -> dict[str, Any]: + """Entry point: return tool_name → callable mapping.""" + if server_context is None: + server_context = {} + db_manager = server_context.get("db_manager") + if db_manager is None: + return {} + return { + "otel_query_spans": _make_query_spans_handler(db_manager), + "otel_list_services": _make_list_services_handler(db_manager), + "otel_cross_layer_query": _make_cross_layer_handler(db_manager), + } diff --git a/plugins/cgc-plugin-otel/src/cgc_plugin_otel/neo4j_writer.py b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/neo4j_writer.py new file mode 100644 index 00000000..6151226b --- /dev/null +++ b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/neo4j_writer.py @@ -0,0 +1,197 @@ +""" +Async Neo4j writer for the OTEL plugin. + +Batches incoming span dicts and flushes them periodically to Neo4j, +with a dead-letter queue for retries during database unavailability. +""" +from __future__ import annotations + +import asyncio +import logging +from typing import Any + +logger = logging.getLogger(__name__) + +_BATCH_SIZE = 100 +_FLUSH_TIMEOUT_S = 5.0 +_QUEUE_MAXSIZE = 10_000 +_DLQ_MAXSIZE = 100_000 + +# --------------------------------------------------------------------------- +# Cypher templates +# --------------------------------------------------------------------------- + +_MERGE_SERVICE = """ +MERGE (s:Service {name: $service_name}) +ON CREATE SET s.first_seen = datetime() +ON MATCH SET s.last_seen = datetime() +""" + +_MERGE_TRACE = """ +MERGE (t:Trace {trace_id: $trace_id}) +ON CREATE SET t.first_seen = datetime() +""" + +_MERGE_SPAN = """ +MERGE (sp:Span {span_id: $span_id}) +ON CREATE SET + sp.trace_id = $trace_id, + sp.name = $name, + sp.span_kind = $span_kind, + sp.service_name = $service_name, + sp.http_route = $http_route, + sp.http_method = $http_method, + sp.class_name = $class_name, + sp.function_name = $function_name, + sp.fqn = $fqn, + sp.db_statement = $db_statement, + sp.db_system = $db_system, + sp.peer_service = $peer_service, + sp.duration_ms = $duration_ms, + sp.start_time_ns = $start_time_ns, + sp.end_time_ns = $end_time_ns, + sp.first_seen = datetime() +ON MATCH SET + sp.observation_count = coalesce(sp.observation_count, 0) + 1, + sp.last_seen = datetime() +""" + +_LINK_SPAN_TO_TRACE = """ +MATCH (sp:Span {span_id: $span_id}), (t:Trace {trace_id: $trace_id}) +MERGE (sp)-[:PART_OF]->(t) +""" + +_LINK_SPAN_TO_SERVICE = """ +MATCH (sp:Span {span_id: $span_id}), (s:Service {name: $service_name}) +MERGE (sp)-[:ORIGINATED_FROM]->(s) +""" + +_LINK_PARENT_SPAN = """ +MATCH (child:Span {span_id: $span_id}), (parent:Span {span_id: $parent_span_id}) +MERGE (child)-[:CHILD_OF]->(parent) +""" + +_LINK_CROSS_SERVICE = """ +MATCH (sp:Span {span_id: $span_id}), (svc:Service {name: $peer_service}) +MERGE (sp)-[:CALLS_SERVICE]->(svc) +""" + +_CORRELATE_TO_METHOD = """ +MATCH (sp:Span {span_id: $span_id}) +WHERE sp.fqn IS NOT NULL +MATCH (m:Method {fqn: sp.fqn}) +MERGE (sp)-[:CORRELATES_TO]->(m) +""" + + +class AsyncOtelWriter: + """ + Buffers spans in an asyncio queue and flushes them to Neo4j in batches. + + Usage:: + + writer = AsyncOtelWriter(db_manager) + asyncio.create_task(writer.run()) # start background flush loop + await writer.enqueue(span_dict) + """ + + def __init__(self, db_manager: Any) -> None: + self._db = db_manager + self._queue: asyncio.Queue[dict] = asyncio.Queue(maxsize=_QUEUE_MAXSIZE) + self._dlq: asyncio.Queue[dict] = asyncio.Queue(maxsize=_DLQ_MAXSIZE) + self._running = False + + async def enqueue(self, span: dict) -> None: + """Add a span to the processing queue, dropping if full.""" + try: + self._queue.put_nowait(span) + except asyncio.QueueFull: + logger.warning("OTEL span queue full — dropping span %s", span.get("span_id")) + + async def run(self) -> None: + """Background task: collect batches and flush.""" + self._running = True + logger.info("AsyncOtelWriter started") + while self._running: + batch = await self._collect_batch() + if batch: + await self._flush_batch(batch) + await self._retry_dlq() + + async def stop(self) -> None: + self._running = False + # Drain remaining items + batch: list[dict] = [] + while not self._queue.empty(): + try: + batch.append(self._queue.get_nowait()) + except asyncio.QueueEmpty: + break + if batch: + await self._flush_batch(batch) + + async def write_batch(self, spans: list[dict]) -> None: + """Write a list of span dicts directly (used in tests and integration).""" + await self._flush_batch(spans) + + # ------------------------------------------------------------------ + # Internal + # ------------------------------------------------------------------ + + async def _collect_batch(self) -> list[dict]: + batch: list[dict] = [] + try: + # Wait for first item + span = await asyncio.wait_for(self._queue.get(), timeout=_FLUSH_TIMEOUT_S) + batch.append(span) + except asyncio.TimeoutError: + return batch + + # Drain up to batch size + while len(batch) < _BATCH_SIZE: + try: + batch.append(self._queue.get_nowait()) + except asyncio.QueueEmpty: + break + return batch + + async def _flush_batch(self, spans: list[dict]) -> None: + try: + driver = self._db.get_driver() + async with driver.session() as session: + for span in spans: + await self._write_span(session, span) + logger.debug("Flushed %d spans to Neo4j", len(spans)) + except Exception as exc: + logger.error("Neo4j flush failed (%s) — moving %d spans to DLQ", exc, len(spans)) + for span in spans: + try: + self._dlq.put_nowait(span) + except asyncio.QueueFull: + logger.warning("DLQ full — permanently dropping span %s", span.get("span_id")) + + async def _write_span(self, session: Any, span: dict) -> None: + await session.run(_MERGE_SERVICE, service_name=span["service_name"]) + await session.run(_MERGE_TRACE, trace_id=span["trace_id"]) + await session.run(_MERGE_SPAN, **span) + await session.run(_LINK_SPAN_TO_TRACE, span_id=span["span_id"], trace_id=span["trace_id"]) + await session.run(_LINK_SPAN_TO_SERVICE, span_id=span["span_id"], service_name=span["service_name"]) + if span.get("parent_span_id"): + await session.run(_LINK_PARENT_SPAN, span_id=span["span_id"], parent_span_id=span["parent_span_id"]) + if span.get("cross_service") and span.get("peer_service"): + await session.run(_LINK_CROSS_SERVICE, span_id=span["span_id"], peer_service=span["peer_service"]) + if span.get("fqn"): + await session.run(_CORRELATE_TO_METHOD, span_id=span["span_id"]) + + async def _retry_dlq(self) -> None: + if self._dlq.empty(): + return + retry_batch: list[dict] = [] + while len(retry_batch) < _BATCH_SIZE and not self._dlq.empty(): + try: + retry_batch.append(self._dlq.get_nowait()) + except asyncio.QueueEmpty: + break + if retry_batch: + logger.info("Retrying %d spans from DLQ", len(retry_batch)) + await self._flush_batch(retry_batch) diff --git a/plugins/cgc-plugin-otel/src/cgc_plugin_otel/receiver.py b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/receiver.py new file mode 100644 index 00000000..b9cfbe05 --- /dev/null +++ b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/receiver.py @@ -0,0 +1,166 @@ +""" +OTLP gRPC receiver for the OTEL plugin. + +Listens for OpenTelemetry trace exports (ExportTraceServiceRequest) and +queues parsed spans for batch writing to Neo4j. + +Requires: + grpcio>=1.57.0 + opentelemetry-proto>=0.43b0 + +Start standalone:: + + python -m cgc_plugin_otel.receiver +""" +from __future__ import annotations + +import asyncio +import logging +import os +import signal +import sys +from typing import Any + +logger = logging.getLogger(__name__) + +_DEFAULT_PORT = int(os.environ.get("OTEL_RECEIVER_PORT", "5317")) +_FILTER_ROUTES = [r.strip() for r in os.environ.get("OTEL_FILTER_ROUTES", "/health,/metrics,/ping").split(",") if r.strip()] + + +def _span_kind_name(kind_int: int) -> str: + kinds = {0: "UNSPECIFIED", 1: "INTERNAL", 2: "SERVER", 3: "CLIENT", 4: "PRODUCER", 5: "CONSUMER"} + return kinds.get(kind_int, "UNSPECIFIED") + + +def _attrs_to_dict(attributes: Any) -> dict: + """Convert protobuf KeyValue list to a plain dict.""" + result: dict = {} + for kv in attributes: + val = kv.value + if val.HasField("string_value"): + result[kv.key] = val.string_value + elif val.HasField("int_value"): + result[kv.key] = val.int_value + elif val.HasField("double_value"): + result[kv.key] = val.double_value + elif val.HasField("bool_value"): + result[kv.key] = val.bool_value + return result + + +class OTLPSpanReceiver: + """ + gRPC servicer implementing the OpenTelemetry TraceService.Export RPC. + + Depends on generated protobuf stubs from ``opentelemetry-proto``. + Import failures are caught at startup; if gRPC is not installed the + plugin still loads but logs a warning. + """ + + def __init__(self, writer: Any, filter_routes: list[str] | None = None) -> None: + self._writer = writer + self._filter_routes = filter_routes or _FILTER_ROUTES + + def Export(self, request: Any, context: Any) -> Any: + """Handle ExportTraceServiceRequest — called by gRPC framework.""" + try: + from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( + ExportTraceServiceResponse, + ) + except ImportError: + logger.error("opentelemetry-proto not installed — cannot process spans") + return None # type: ignore[return-value] + + from cgc_plugin_otel.span_processor import build_span_dict, should_filter_span + + for resource_spans in request.resource_spans: + service_name = "unknown" + for attr in resource_spans.resource.attributes: + if attr.key == "service.name": + service_name = attr.value.string_value + break + + for scope_spans in resource_spans.scope_spans: + for span in scope_spans.spans: + attrs = _attrs_to_dict(span.attributes) + if should_filter_span(attrs, self._filter_routes): + continue + + span_dict = build_span_dict( + span_id=span.span_id.hex(), + trace_id=span.trace_id.hex(), + parent_span_id=span.parent_span_id.hex() if span.parent_span_id else None, + name=span.name, + span_kind=_span_kind_name(span.kind), + start_time_ns=span.start_time_unix_nano, + end_time_ns=span.end_time_unix_nano, + attributes=attrs, + service_name=service_name, + ) + # Schedule on the event loop — receiver runs in gRPC thread pool + loop = asyncio.get_event_loop() + asyncio.run_coroutine_threadsafe(self._writer.enqueue(span_dict), loop) + + return ExportTraceServiceResponse() + + +def main() -> None: + """Start the OTLP gRPC receiver and the async writer background task.""" + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s") + + try: + import grpc + from opentelemetry.proto.collector.trace.v1 import trace_service_pb2_grpc + except ImportError as exc: + logger.error("Cannot start OTEL receiver — missing dependency: %s", exc) + sys.exit(1) + + # Import db_manager from CGC core + try: + from codegraphcontext.core import get_database_manager + db_manager = get_database_manager() + except Exception as exc: + logger.error("Cannot connect to database: %s", exc) + sys.exit(1) + + from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter + + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + writer = AsyncOtelWriter(db_manager) + servicer = OTLPSpanReceiver(writer) + + server = grpc.server( + grpc.experimental.aio.server() if False else # type: ignore[misc] + grpc.server(grpc.experimental.insecure_channel_credentials()) # type: ignore[misc] + ) + + # Simpler: use sync gRPC server with ThreadPoolExecutor + server = grpc.server(__import__("concurrent.futures", fromlist=["ThreadPoolExecutor"]).ThreadPoolExecutor(max_workers=4)) + trace_service_pb2_grpc.add_TraceServiceServicer_to_server(servicer, server) + server.add_insecure_port(f"[::]:{_DEFAULT_PORT}") + server.start() + logger.info("OTLP gRPC receiver listening on port %d", _DEFAULT_PORT) + + writer_task = loop.create_task(writer.run()) + + def _shutdown(signum: int, frame: Any) -> None: + logger.info("Shutting down OTEL receiver…") + server.stop(grace=5) + loop.call_soon_threadsafe(writer_task.cancel) + + signal.signal(signal.SIGTERM, _shutdown) + signal.signal(signal.SIGINT, _shutdown) + + try: + loop.run_forever() + except KeyboardInterrupt: + pass + finally: + loop.run_until_complete(writer.stop()) + loop.close() + + +if __name__ == "__main__": + main() diff --git a/plugins/cgc-plugin-otel/src/cgc_plugin_otel/span_processor.py b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/span_processor.py new file mode 100644 index 00000000..78194bcf --- /dev/null +++ b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/span_processor.py @@ -0,0 +1,103 @@ +""" +Pure-logic span processing for the OTEL plugin. + +No gRPC or database dependencies — these functions transform raw span attributes +into typed dicts that the writer can persist to the graph. +""" +from __future__ import annotations + + +def extract_php_context(span_attrs: dict) -> dict: + """ + Parse PHP-specific OpenTelemetry attributes from a span attribute dict. + + Returns a typed dict with all known PHP context keys. Missing keys are + returned as ``None`` rather than raising ``KeyError``. + """ + return { + "namespace": span_attrs.get("code.namespace"), + "function": span_attrs.get("code.function"), + "http_route": span_attrs.get("http.route"), + "http_method": span_attrs.get("http.method"), + "db_statement": span_attrs.get("db.statement"), + "db_system": span_attrs.get("db.system"), + "peer_service": span_attrs.get("peer.service"), + } + + +def build_fqn(namespace: str | None, function: str | None) -> str | None: + """ + Build a fully-qualified name from PHP code.namespace and code.function. + + Returns ``None`` if either component is missing. + """ + if namespace is None or function is None: + return None + return f"{namespace}::{function}" + + +def is_cross_service_span(span_kind: str, span_attrs: dict) -> bool: + """ + Return True when this span represents a call from one service to another. + + A span is cross-service when its kind is CLIENT and ``peer.service`` is set. + """ + return span_kind == "CLIENT" and bool(span_attrs.get("peer.service")) + + +def should_filter_span(span_attrs: dict, filter_routes: list[str]) -> bool: + """ + Return True when the span's HTTP route matches a configured noise filter. + + Spans without an ``http.route`` attribute are never filtered. + """ + if not filter_routes: + return False + route = span_attrs.get("http.route") + if route is None: + return False + return route in filter_routes + + +def build_span_dict( + *, + span_id: str, + trace_id: str, + parent_span_id: str | None, + name: str, + span_kind: str, + start_time_ns: int, + end_time_ns: int, + attributes: dict, + service_name: str, +) -> dict: + """ + Build a normalised span dict ready for Neo4j persistence. + + Duration is converted from nanoseconds to milliseconds. + """ + duration_ms = (end_time_ns - start_time_ns) / 1_000_000 + + php_ctx = extract_php_context(attributes) + fqn = build_fqn(php_ctx["namespace"], php_ctx["function"]) + + return { + "span_id": span_id, + "trace_id": trace_id, + "parent_span_id": parent_span_id, + "name": name, + "span_kind": span_kind, + "service_name": service_name, + "start_time_ns": start_time_ns, + "end_time_ns": end_time_ns, + "duration_ms": duration_ms, + "http_route": php_ctx["http_route"], + "http_method": php_ctx["http_method"], + "class_name": php_ctx["namespace"], + "function_name": php_ctx["function"], + "fqn": fqn, + "db_statement": php_ctx["db_statement"], + "db_system": php_ctx["db_system"], + "peer_service": php_ctx["peer_service"], + "cross_service": is_cross_service_span(span_kind, attributes), + } diff --git a/plugins/cgc-plugin-stub/pyproject.toml b/plugins/cgc-plugin-stub/pyproject.toml new file mode 100644 index 00000000..71d73e2d --- /dev/null +++ b/plugins/cgc-plugin-stub/pyproject.toml @@ -0,0 +1,31 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "cgc-plugin-stub" +version = "0.1.0" +description = "Minimal stub plugin for testing the CGC plugin system" +readme = "README.md" +requires-python = ">=3.10" +authors = [ + { name = "CodeGraphContext Contributors" } +] +dependencies = [ + "typer[all]>=0.9.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.4.0", +] + +[project.entry-points."cgc_cli_plugins"] +stub = "cgc_plugin_stub.cli:get_plugin_commands" + +[project.entry-points."cgc_mcp_plugins"] +stub = "cgc_plugin_stub.mcp_tools:get_mcp_tools" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["cgc_plugin_stub*"] diff --git a/plugins/cgc-plugin-stub/src/cgc_plugin_stub/__init__.py b/plugins/cgc-plugin-stub/src/cgc_plugin_stub/__init__.py new file mode 100644 index 00000000..d4185be7 --- /dev/null +++ b/plugins/cgc-plugin-stub/src/cgc_plugin_stub/__init__.py @@ -0,0 +1,8 @@ +"""Stub plugin for testing the CGC plugin system.""" + +PLUGIN_METADATA = { + "name": "cgc-plugin-stub", + "version": "0.1.0", + "cgc_version_constraint": ">=0.1.0", + "description": "Minimal stub plugin for testing CGC plugin discovery and loading.", +} diff --git a/plugins/cgc-plugin-stub/src/cgc_plugin_stub/cli.py b/plugins/cgc-plugin-stub/src/cgc_plugin_stub/cli.py new file mode 100644 index 00000000..085f9270 --- /dev/null +++ b/plugins/cgc-plugin-stub/src/cgc_plugin_stub/cli.py @@ -0,0 +1,15 @@ +"""CLI command group contributed by the stub plugin.""" +import typer + +stub_app = typer.Typer(name="stub", help="Stub plugin commands (for testing).") + + +@stub_app.command() +def hello(): + """Echo a greeting from the stub plugin.""" + typer.echo("Hello from stub plugin") + + +def get_plugin_commands() -> tuple[str, typer.Typer]: + """Entry point: return (command_name, typer_app).""" + return ("stub", stub_app) diff --git a/plugins/cgc-plugin-stub/src/cgc_plugin_stub/mcp_tools.py b/plugins/cgc-plugin-stub/src/cgc_plugin_stub/mcp_tools.py new file mode 100644 index 00000000..fff439e8 --- /dev/null +++ b/plugins/cgc-plugin-stub/src/cgc_plugin_stub/mcp_tools.py @@ -0,0 +1,37 @@ +"""MCP tools contributed by the stub plugin.""" +from __future__ import annotations + +from typing import Any + + +_TOOLS: dict[str, dict] = { + "stub_hello": { + "name": "stub_hello", + "description": "Say hello — stub plugin smoke test tool.", + "inputSchema": { + "type": "object", + "properties": { + "name": { + "type": "string", + "description": "Name to greet", + "default": "World", + } + }, + "required": [], + }, + } +} + + +def _handle_stub_hello(name: str = "World", **_kwargs: Any) -> dict: + return {"greeting": f"Hello {name}"} + + +def get_mcp_tools(server_context: dict | None = None) -> dict[str, dict]: + """Entry point: return tool_name → ToolDefinition mapping.""" + return _TOOLS + + +def get_mcp_handlers(server_context: dict | None = None) -> dict[str, Any]: + """Entry point: return tool_name → callable mapping.""" + return {"stub_hello": _handle_stub_hello} diff --git a/plugins/cgc-plugin-xdebug/Dockerfile b/plugins/cgc-plugin-xdebug/Dockerfile new file mode 100644 index 00000000..6791d1ed --- /dev/null +++ b/plugins/cgc-plugin-xdebug/Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.12-slim + +RUN groupadd -r cgc && useradd -r -g cgc cgc + +WORKDIR /app + +COPY pyproject.toml README.md ./ +COPY src/ ./src/ + +RUN pip install --no-cache-dir -e . && \ + pip install --no-cache-dir "typer[all]>=0.9.0" "neo4j>=5.15.0" || true + +USER cgc + +EXPOSE 9003 + +HEALTHCHECK --interval=30s --timeout=10s --start-period=10s --retries=3 \ + CMD python -c "import socket; socket.socket(socket.AF_INET, socket.SOCK_STREAM)" || exit 1 + +# CGC_PLUGIN_XDEBUG_ENABLED must be set to 'true' at runtime +CMD ["python", "-m", "cgc_plugin_xdebug.dbgp_server"] diff --git a/plugins/cgc-plugin-xdebug/pyproject.toml b/plugins/cgc-plugin-xdebug/pyproject.toml new file mode 100644 index 00000000..c338c412 --- /dev/null +++ b/plugins/cgc-plugin-xdebug/pyproject.toml @@ -0,0 +1,35 @@ +[build-system] +requires = ["setuptools>=61.0"] +build-backend = "setuptools.build_meta" + +[project] +name = "cgc-plugin-xdebug" +version = "0.1.0" +description = "Xdebug DBGp listener plugin for CodeGraphContext (dev/staging only)" +readme = "README.md" +requires-python = ">=3.10" +authors = [ + { name = "CodeGraphContext Contributors" } +] +dependencies = [ + "codegraphcontext>=0.3.0", + "typer[all]>=0.9.0", + "neo4j>=5.15.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.4.0", + "pytest-asyncio>=0.21.0", + "pytest-mock>=3.11.0", +] + +[project.entry-points."cgc_cli_plugins"] +xdebug = "cgc_plugin_xdebug.cli:get_plugin_commands" + +[project.entry-points."cgc_mcp_plugins"] +xdebug = "cgc_plugin_xdebug.mcp_tools:get_mcp_tools" + +[tool.setuptools.packages.find] +where = ["src"] +include = ["cgc_plugin_xdebug*"] diff --git a/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/__init__.py b/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/__init__.py new file mode 100644 index 00000000..f7d7b1e7 --- /dev/null +++ b/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/__init__.py @@ -0,0 +1,16 @@ +"""Xdebug plugin for CodeGraphContext — captures PHP call stacks via DBGp and writes them to the graph. + +NOTE: This plugin is intended for development and staging environments only. +It must be explicitly enabled via CGC_PLUGIN_XDEBUG_ENABLED=true. +""" + +PLUGIN_METADATA = { + "name": "cgc-plugin-xdebug", + "version": "0.1.0", + "cgc_version_constraint": ">=0.1.0", + "description": ( + "Runs a TCP DBGp listener, captures PHP call stacks from Xdebug, " + "deduplicates chains, and writes StackFrame nodes to the code graph. " + "Development/staging only — requires CGC_PLUGIN_XDEBUG_ENABLED=true." + ), +} diff --git a/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/cli.py b/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/cli.py new file mode 100644 index 00000000..f2cb444f --- /dev/null +++ b/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/cli.py @@ -0,0 +1,90 @@ +"""CLI command group contributed by the Xdebug plugin.""" +from __future__ import annotations + +import os +import threading +import typer +from typing import Optional + +xdebug_app = typer.Typer(name="xdebug", help="Xdebug DBGp call-stack capture commands.") + +_server_thread: threading.Thread | None = None + + +@xdebug_app.command("start") +def start( + host: str = typer.Option("0.0.0.0", "--host", help="Bind address"), + port: int = typer.Option(9003, "--port", help="DBGp listen port"), +): + """Start the Xdebug DBGp TCP listener (requires CGC_PLUGIN_XDEBUG_ENABLED=true).""" + global _server_thread + if os.environ.get("CGC_PLUGIN_XDEBUG_ENABLED", "").lower() != "true": + typer.echo("CGC_PLUGIN_XDEBUG_ENABLED is not set to 'true' — refusing to start.", err=True) + raise typer.Exit(1) + + try: + from codegraphcontext.core import get_database_manager + db = get_database_manager() + except Exception as e: + typer.echo(f"Database unavailable: {e}", err=True) + raise typer.Exit(1) + + from cgc_plugin_xdebug.neo4j_writer import XdebugWriter + from cgc_plugin_xdebug.dbgp_server import DBGpServer + + writer = XdebugWriter(db) + server = DBGpServer(writer, host=host, port=port) + + _server_thread = threading.Thread(target=server.listen, daemon=True, name="xdebug-dbgp") + _server_thread.start() + typer.echo(f"Xdebug DBGp listener started on {host}:{port} (Ctrl-C to stop)") + try: + _server_thread.join() + except KeyboardInterrupt: + server.stop() + typer.echo("\nXdebug listener stopped.") + + +@xdebug_app.command("status") +def status(): + """Show Xdebug listener configuration.""" + enabled = os.environ.get("CGC_PLUGIN_XDEBUG_ENABLED", "false") + port = os.environ.get("XDEBUG_LISTEN_PORT", "9003") + typer.echo(f"CGC_PLUGIN_XDEBUG_ENABLED: {enabled}") + typer.echo(f"XDEBUG_LISTEN_PORT: {port}") + if enabled.lower() != "true": + typer.echo("Listener is NOT enabled.") + else: + typer.echo("Run 'cgc xdebug start' to start the listener.") + + +@xdebug_app.command("list-chains") +def list_chains( + limit: int = typer.Option(20, "--limit", help="Maximum chains to display"), +): + """List the most-observed call stack chains.""" + try: + from codegraphcontext.core import get_database_manager + db = get_database_manager() + driver = db.get_driver() + with driver.session() as session: + rows = session.run( + "MATCH (sf:StackFrame) WHERE sf.observation_count > 0 " + "RETURN sf.fqn AS fqn, sf.observation_count AS count " + "ORDER BY count DESC LIMIT $limit", + limit=limit, + ).data() + except Exception as e: + typer.echo(f"Error: {e}", err=True) + raise typer.Exit(1) + + if not rows: + typer.echo("No chains recorded.") + return + for row in rows: + typer.echo(f"{row['count']:>6}x {row['fqn']}") + + +def get_plugin_commands() -> tuple[str, typer.Typer]: + """Entry point: return (command_name, typer_app).""" + return ("xdebug", xdebug_app) diff --git a/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/dbgp_server.py b/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/dbgp_server.py new file mode 100644 index 00000000..5add5402 --- /dev/null +++ b/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/dbgp_server.py @@ -0,0 +1,223 @@ +""" +DBGp TCP listener for the Xdebug plugin. + +Implements a minimal DBGp debug client that: + 1. Accepts inbound Xdebug connections on a configurable TCP port. + 2. Sends a ``stack_get`` command to retrieve the call stack. + 3. Parses the XML response into a list of frame dicts. + 4. Delegates persistence to XdebugWriter. + +The server only starts when CGC_PLUGIN_XDEBUG_ENABLED=true. +Uses only Python stdlib (socket, xml.etree.ElementTree, hashlib). +""" +from __future__ import annotations + +import hashlib +import logging +import os +import socket +import threading +import xml.etree.ElementTree as ET +from typing import Any + +logger = logging.getLogger(__name__) + +_DBGP_NS = "urn:debugger_protocol_v1" +_DEFAULT_HOST = os.environ.get("XDEBUG_LISTEN_HOST", "0.0.0.0") +_DEFAULT_PORT = int(os.environ.get("XDEBUG_LISTEN_PORT", "9003")) +_ENABLED_ENV = "CGC_PLUGIN_XDEBUG_ENABLED" + + +# --------------------------------------------------------------------------- +# Pure-logic helpers (no I/O — tested directly) +# --------------------------------------------------------------------------- + +def parse_stack_xml(xml_str: str) -> list[dict]: + """ + Parse a DBGp ``stack_get`` XML response into a list of frame dicts. + + Returns frames ordered by ``level`` (ascending, 0 = current frame). + The ``file://`` scheme prefix is stripped from filenames. + """ + try: + root = ET.fromstring(xml_str) + except ET.ParseError as exc: + logger.warning("Failed to parse DBGp XML: %s", exc) + return [] + + frames: list[dict] = [] + for stack_el in root.findall(f"{{{_DBGP_NS}}}stack") + root.findall("stack"): + filename = stack_el.get("filename", "") + if filename.startswith("file://"): + filename = filename[7:] + + frames.append({ + "where": stack_el.get("where", ""), + "level": int(stack_el.get("level", 0)), + "filename": filename, + "lineno": int(stack_el.get("lineno", 0)), + }) + + return sorted(frames, key=lambda f: f["level"]) + + +def compute_chain_hash(frames: list[dict]) -> str: + """ + Compute a deterministic SHA-256 hash for a call stack chain. + + Two identical chains (same where/filename/lineno sequence) produce the + same hash, enabling efficient deduplication. + """ + key = "|".join( + f"{f.get('where','')}:{f.get('filename','')}:{f.get('lineno',0)}" + for f in frames + ) + return hashlib.sha256(key.encode()).hexdigest() + + +def build_frame_id(class_name: str, method_name: str, file_path: str, lineno: int) -> str: + """ + Build a deterministic unique frame identifier string. + + The ID is a SHA-256 hex digest of the four components, ensuring + stability across restarts. + """ + key = f"{class_name}::{method_name}::{file_path}::{lineno}" + return hashlib.sha256(key.encode()).hexdigest() + + +def _parse_where(where: str) -> tuple[str | None, str | None]: + """Split a DBGp 'where' string (Class->method or Class::method) into (class, method).""" + for sep in ("->", "::"): + if sep in where: + parts = where.rsplit(sep, 1) + return parts[0], parts[1] + return None, where or None + + +# --------------------------------------------------------------------------- +# TCP Server +# --------------------------------------------------------------------------- + +class DBGpServer: + """ + Minimal TCP DBGp server that captures PHP call stacks. + + Only starts when ``CGC_PLUGIN_XDEBUG_ENABLED=true``. + """ + + def __init__(self, writer: Any, host: str = _DEFAULT_HOST, port: int = _DEFAULT_PORT) -> None: + self._writer = writer + self._host = host + self._port = port + self._running = False + self._sock: socket.socket | None = None + + def is_enabled(self) -> bool: + return os.environ.get(_ENABLED_ENV, "").lower() == "true" + + def listen(self) -> None: + """Start the TCP listener (blocking). Requires XDEBUG_ENABLED env var.""" + if not self.is_enabled(): + logger.warning( + "Xdebug DBGp server NOT started — set %s=true to enable", _ENABLED_ENV + ) + return + + self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._sock.bind((self._host, self._port)) + self._sock.listen(10) + self._running = True + logger.info("DBGp server listening on %s:%d", self._host, self._port) + + while self._running: + try: + conn, addr = self._sock.accept() + logger.debug("Xdebug connection from %s", addr) + t = threading.Thread(target=self._handle_connection, args=(conn,), daemon=True) + t.start() + except OSError: + break # socket closed + + def stop(self) -> None: + self._running = False + if self._sock: + self._sock.close() + + def _handle_connection(self, conn: socket.socket) -> None: + try: + self._process_session(conn) + except Exception as exc: + logger.debug("DBGp session error: %s", exc) + finally: + conn.close() + + def _process_session(self, conn: socket.socket) -> None: + # Read the init packet (Xdebug sends XML on connect) + _init_xml = self._recv_packet(conn) + + seq = 1 + # Send run to start execution + self._send_cmd(conn, f"run -i {seq}") + seq += 1 + + while True: + # Request the current call stack + self._send_cmd(conn, f"stack_get -i {seq}") + seq += 1 + + response = self._recv_packet(conn) + if not response: + break + + frames = parse_stack_xml(response) + if frames: + self._writer.write_chain(frames) + + # Send run to continue to next breakpoint / end of script + self._send_cmd(conn, f"run -i {seq}") + seq += 1 + + # Check if execution ended + ack = self._recv_packet(conn) + if not ack or "status=\"stopped\"" in ack or "status='stopped'" in ack: + break + + @staticmethod + def _send_cmd(conn: socket.socket, cmd: str) -> None: + data = (cmd + "\0").encode() + conn.sendall(data) + + @staticmethod + def _recv_packet(conn: socket.socket) -> str: + """Read a DBGp length-prefixed null-terminated packet.""" + chunks: list[bytes] = [] + # Read the length digits up to the first \0 + length_bytes = bytearray() + while True: + byte = conn.recv(1) + if not byte: + return "" + if byte == b"\0": + break + length_bytes.extend(byte) + + if not length_bytes: + return "" + + try: + length = int(length_bytes) + except ValueError: + return "" + + # Read the XML body (length bytes + trailing \0) + remaining = length + 1 + while remaining > 0: + chunk = conn.recv(min(remaining, 4096)) + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + + return b"".join(chunks).rstrip(b"\0").decode("utf-8", errors="replace") diff --git a/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/mcp_tools.py b/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/mcp_tools.py new file mode 100644 index 00000000..ce4ee345 --- /dev/null +++ b/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/mcp_tools.py @@ -0,0 +1,101 @@ +"""MCP tools contributed by the Xdebug plugin.""" +from __future__ import annotations + +from typing import Any + +_TOOLS: dict[str, dict] = { + "xdebug_list_chains": { + "name": "xdebug_list_chains", + "description": ( + "List the most-observed PHP call stack chains captured by Xdebug. " + "Returns StackFrame nodes ordered by observation count." + ), + "inputSchema": { + "type": "object", + "properties": { + "limit": {"type": "integer", "default": 20, "description": "Max results"}, + "min_observations": { + "type": "integer", + "default": 1, + "description": "Minimum observation count to include", + }, + }, + "required": [], + }, + }, + "xdebug_query_chain": { + "name": "xdebug_query_chain", + "description": ( + "Query the call stack chains that include a specific class or method. " + "Returns StackFrame nodes with their CALLED_BY chain." + ), + "inputSchema": { + "type": "object", + "properties": { + "class_name": {"type": "string", "description": "PHP class name (partial match)"}, + "method_name": {"type": "string", "description": "PHP method name (partial match)"}, + "limit": {"type": "integer", "default": 10}, + }, + "required": [], + }, + }, +} + + +def _make_list_chains_handler(db_manager: Any): + def handle(limit: int = 20, min_observations: int = 1, **_: Any) -> dict: + driver = db_manager.get_driver() + with driver.session() as session: + rows = session.run( + "MATCH (sf:StackFrame) WHERE sf.observation_count >= $min_obs " + "RETURN sf.fqn AS fqn, sf.file_path AS file, sf.lineno AS lineno, " + "sf.observation_count AS observations " + "ORDER BY observations DESC LIMIT $limit", + min_obs=min_observations, + limit=limit, + ).data() + return {"chains": rows} + return handle + + +def _make_query_chain_handler(db_manager: Any): + def handle(class_name: str | None = None, method_name: str | None = None, limit: int = 10, **_: Any) -> dict: + where_parts = [] + params: dict = {"limit": limit} + if class_name: + where_parts.append("sf.class_name CONTAINS $class_name") + params["class_name"] = class_name + if method_name: + where_parts.append("sf.method_name CONTAINS $method_name") + params["method_name"] = method_name + + where = ("WHERE " + " AND ".join(where_parts)) if where_parts else "" + cypher = ( + f"MATCH (sf:StackFrame) {where} " + "OPTIONAL MATCH (sf)-[:CALLED_BY*1..5]->(caller:StackFrame) " + "RETURN sf.fqn AS root_fqn, collect(caller.fqn) AS call_chain, " + "sf.observation_count AS observations " + "ORDER BY observations DESC LIMIT $limit" + ) + driver = db_manager.get_driver() + with driver.session() as session: + return {"results": session.run(cypher, **params).data()} + return handle + + +def get_mcp_tools(server_context: dict | None = None) -> dict[str, dict]: + """Entry point: return tool_name → ToolDefinition mapping.""" + return _TOOLS + + +def get_mcp_handlers(server_context: dict | None = None) -> dict[str, Any]: + """Entry point: return tool_name → callable mapping.""" + if server_context is None: + server_context = {} + db_manager = server_context.get("db_manager") + if db_manager is None: + return {} + return { + "xdebug_list_chains": _make_list_chains_handler(db_manager), + "xdebug_query_chain": _make_query_chain_handler(db_manager), + } diff --git a/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/neo4j_writer.py b/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/neo4j_writer.py new file mode 100644 index 00000000..e8d38854 --- /dev/null +++ b/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/neo4j_writer.py @@ -0,0 +1,142 @@ +""" +Neo4j writer for the Xdebug plugin. + +Persists PHP call stack chains as StackFrame nodes in the graph, +with LRU-based deduplication to avoid redundant writes. +""" +from __future__ import annotations + +import logging +import os +from typing import Any + +from cgc_plugin_xdebug.dbgp_server import ( + compute_chain_hash, + build_frame_id, + _parse_where, +) + +logger = logging.getLogger(__name__) + +_DEDUP_CACHE_SIZE = int(os.environ.get("XDEBUG_DEDUP_CACHE_SIZE", "10000")) + +# --------------------------------------------------------------------------- +# Cypher templates +# --------------------------------------------------------------------------- + +_MERGE_FRAME = """ +MERGE (sf:StackFrame {frame_id: $frame_id}) +ON CREATE SET + sf.fqn = $fqn, + sf.class_name = $class_name, + sf.method_name = $method_name, + sf.file_path = $file_path, + sf.lineno = $lineno, + sf.observation_count = 1, + sf.first_seen = datetime() +ON MATCH SET + sf.observation_count = coalesce(sf.observation_count, 0) + 1, + sf.last_seen = datetime() +""" + +_LINK_CALLED_BY = """ +MATCH (callee:StackFrame {frame_id: $callee_id}), (caller:StackFrame {frame_id: $caller_id}) +MERGE (callee)-[:CALLED_BY]->(caller) +""" + +_LINK_RESOLVES_TO = """ +MATCH (sf:StackFrame {frame_id: $frame_id}) +WHERE sf.fqn IS NOT NULL +MATCH (m:Method {fqn: sf.fqn}) +MERGE (sf)-[:RESOLVES_TO]->(m) +""" + +_INCREMENT_OBSERVATION = """ +MATCH (sf:StackFrame {frame_id: $frame_id}) +SET sf.observation_count = coalesce(sf.observation_count, 0) + 1, + sf.last_seen = datetime() +""" + + +class XdebugWriter: + """ + Writes Xdebug call stack chains to Neo4j with LRU deduplication. + + When the same chain hash is seen again the writer skips a full MERGE + and only increments the observation_count on the root frame. + """ + + def __init__(self, db_manager: Any, cache_size: int = _DEDUP_CACHE_SIZE) -> None: + self._db = db_manager + self._cache: dict[str, int] = {} # hash → root frame_id + self._cache_size = cache_size + + def write_chain(self, frames: list[dict]) -> None: + """ + Persist a call stack chain. + + If the chain was seen before, only increments the root frame's + observation_count; otherwise writes all StackFrame nodes and + CALLED_BY links, then attempts RESOLVES_TO for each frame. + """ + if not frames: + return + + chain_hash = compute_chain_hash(frames) + if chain_hash in self._cache: + root_frame_id = self._cache[chain_hash] + self._increment_observation(root_frame_id) + return + + driver = self._db.get_driver() + with driver.session() as session: + frame_ids: list[str] = [] + for frame in frames: + class_name, method_name = _parse_where(frame.get("where", "")) + fqn = f"{class_name}::{method_name}" if class_name and method_name else None + frame_id = build_frame_id( + class_name or "", + method_name or "", + frame.get("filename", ""), + frame.get("lineno", 0), + ) + session.run( + _MERGE_FRAME, + frame_id=frame_id, + fqn=fqn, + class_name=class_name, + method_name=method_name, + file_path=frame.get("filename"), + lineno=frame.get("lineno", 0), + ) + frame_ids.append(frame_id) + + # CALLED_BY links: frame[n] called by frame[n+1] + for i in range(len(frame_ids) - 1): + session.run( + _LINK_CALLED_BY, + callee_id=frame_ids[i], + caller_id=frame_ids[i + 1], + ) + + # Try to link each frame to a static Method node + for frame_id in frame_ids: + session.run(_LINK_RESOLVES_TO, frame_id=frame_id) + + root_frame_id = frame_ids[0] if frame_ids else "" + self._evict_if_needed() + self._cache[chain_hash] = root_frame_id + + def _increment_observation(self, frame_id: str) -> None: + try: + driver = self._db.get_driver() + with driver.session() as session: + session.run(_INCREMENT_OBSERVATION, frame_id=frame_id) + except Exception as exc: + logger.warning("Failed to increment observation for frame %s: %s", frame_id, exc) + + def _evict_if_needed(self) -> None: + if len(self._cache) >= self._cache_size: + # Evict oldest entry (first inserted key in CPython 3.7+) + oldest = next(iter(self._cache)) + del self._cache[oldest] diff --git a/pyproject.toml b/pyproject.toml index cadc0c08..e4a9c6d9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -15,6 +15,7 @@ classifiers = [ "Topic :: Software Development :: Libraries :: Application Frameworks", ] dependencies = [ + "packaging>=23.0", "neo4j>=5.15.0", "watchdog>=3.0.0", "stdlibs>=2023.11.18", @@ -44,6 +45,21 @@ dev = [ "pytest>=7.4.0", "black>=23.11.0", "pytest-asyncio>=0.21.0", + "pytest-mock>=3.11.0", +] +otel = [ + "cgc-plugin-otel>=0.1.0", +] +xdebug = [ + "cgc-plugin-xdebug>=0.1.0", +] +memory = [ + "cgc-plugin-memory>=0.1.0", +] +all = [ + "cgc-plugin-otel>=0.1.0", + "cgc-plugin-xdebug>=0.1.0", + "cgc-plugin-memory>=0.1.0", ] [project.urls] diff --git a/specs/001-cgc-plugin-extension/checklists/requirements.md b/specs/001-cgc-plugin-extension/checklists/requirements.md new file mode 100644 index 00000000..45d719dd --- /dev/null +++ b/specs/001-cgc-plugin-extension/checklists/requirements.md @@ -0,0 +1,48 @@ +# Specification Quality Checklist: CGC Plugin Extension System + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2026-03-14 +**Feature**: [spec.md](../spec.md) + +## Content Quality + +- [x] No implementation details (languages, frameworks, APIs) — requirements are + user/outcome-focused; technical protocol references (OTEL, DBGp) are domain- + inherent, not avoidable implementation choices; specifics confined to Assumptions +- [x] Focused on user value and business needs +- [x] Written for non-technical stakeholders (with domain-specific protocol names + explained by context) +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No [NEEDS CLARIFICATION] markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria are technology-agnostic (except SC-010 which names K8s primitives + — acceptable since K8s compatibility is the explicit stated goal of the feature) +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified +- [x] Scope is clearly bounded (5 user stories with explicit in/out of scope via + Assumptions section) +- [x] Dependencies and assumptions identified (Assumptions section present) + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows (plugin lifecycle, each of the three plugin + types, and CI/CD pipeline) +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] No implementation details leak into specification (technical protocols are domain + vocabulary, not implementation choices; language/tooling confined to Assumptions) + +## Notes + +- All items passed on first validation iteration. No spec updates required before + `/speckit.plan` or `/speckit.clarify`. +- SC-010 intentionally references Kubernetes primitives because K8s compatibility is + the explicit stated requirement from the feature description; this is not an + implementation leak. +- Protocol names (OTEL/OpenTelemetry, DBGp/Xdebug, MCP) are treated as domain + vocabulary equivalent to naming "REST API" or "OAuth" — they identify the integration + standard, not the implementation approach. diff --git a/specs/001-cgc-plugin-extension/contracts/cicd-pipeline.md b/specs/001-cgc-plugin-extension/contracts/cicd-pipeline.md new file mode 100644 index 00000000..21d423d7 --- /dev/null +++ b/specs/001-cgc-plugin-extension/contracts/cicd-pipeline.md @@ -0,0 +1,149 @@ +# Contract: CI/CD Pipeline for Plugin Service Images + +**Version**: 1.0.0 +**Feature**: 001-cgc-plugin-extension +**Audience**: CGC maintainers and plugin authors contributing container services + +--- + +## 1. Pipeline Triggers + +The shared Docker build pipeline (`docker-publish.yml`) runs on: + +| Trigger | Behavior | +|---|---| +| Push to `main` branch | Build all service images; push with `latest` tag | +| Push of a semver tag (`v*`) | Build all images; push with version tags + `latest` | +| Pull request to `main` | Build all images; smoke test; do NOT push | +| Manual dispatch | Build all images; push with `latest` | + +--- + +## 2. Service Registry + +All container services are declared in `.github/services.json`. This is the only file +that MUST be edited to add or remove a service from the pipeline. + +**Schema**: +```json +[ + { + "name": "cgc-core", + "path": ".", + "dockerfile": "Dockerfile", + "health_check": "version" + }, + { + "name": "cgc-plugin-otel", + "path": "plugins/cgc-plugin-otel", + "dockerfile": "plugins/cgc-plugin-otel/Dockerfile", + "health_check": "grpc_ping" + }, + { + "name": "cgc-plugin-memory", + "path": "plugins/cgc-plugin-memory", + "dockerfile": "plugins/cgc-plugin-memory/Dockerfile", + "health_check": "http_health" + } +] +``` + +| Field | Type | Description | +|---|---|---| +| `name` | string | Image name (used as registry path segment) | +| `path` | string | Docker build context path (relative to repo root) | +| `dockerfile` | string | Path to Dockerfile (relative to repo root) | +| `health_check` | string | Smoke test type: `"version"`, `"grpc_ping"`, `"http_health"` | + +--- + +## 3. Image Tagging Convention + +All images are published to the configured registry under: +`//:` + +Tags produced per build: + +| Event | Tags | +|---|---| +| Tag `v1.2.3` pushed | `1.2.3`, `1.2`, `1`, `latest` | +| Push to `main` | `latest`, `main-` | +| Push to other branch | `-` | +| Pull request | `pr-` (not pushed) | + +--- + +## 4. Smoke Test Types + +Each service MUST declare a `health_check` type. The pipeline runs the corresponding +test against the locally-built image before pushing. + +| Type | Test command | Pass condition | +|---|---|---| +| `version` | `docker run --rm --version` | Exit code 0 | +| `grpc_ping` | `docker run --rm python -c "import grpc; print('ok')"` | Exit code 0 | +| `http_health` | Start container, `curl -f http://localhost:/health` | HTTP 200 | + +A build that fails its smoke test MUST NOT be pushed to the registry. +Other services' builds continue regardless (`fail-fast: false`). + +--- + +## 5. Dockerfile Requirements + +Every service Dockerfile MUST: + +1. Use a minimal base image (e.g. `python:3.12-slim`, NOT `python:3.12`) +2. Run as a non-root user (final `USER` instruction MUST NOT be root) +3. Include a `HEALTHCHECK` instruction that CGC's health_check type can exercise +4. Accept all configuration via environment variables (no credentials in `ENV`) +5. Produce a reproducible build (pin dependency versions) + +**Example `HEALTHCHECK`** for a Python service: +```dockerfile +HEALTHCHECK --interval=30s --timeout=10s --start-period=15s --retries=3 \ + CMD python -c "import sys; sys.exit(0)" || exit 1 +``` + +--- + +## 6. Kubernetes Compatibility Requirements + +Published images MUST be deployable via standard Kubernetes `Deployment` + `Service` +manifests with no special configuration: + +- No `hostNetwork: true` required +- No `privileged: true` required +- All config via environment variables (compatible with `ConfigMap` + `Secret`) +- Readiness and liveness probes derivable from `HEALTHCHECK` +- No persistent volume required for stateless plugin services + (Neo4j connection details passed via env vars) + +Reference K8s manifests are provided in `k8s//` for each service. + +--- + +## 7. Adding a New Service + +To add a new plugin service to the pipeline: + +1. Add the service entry to `.github/services.json` +2. Ensure the plugin directory has a `Dockerfile` satisfying §5 +3. The pipeline automatically picks up the new service on the next run + +No other workflow changes are required. + +--- + +## 8. Registry Configuration + +The target registry is configured via repository secrets/variables: + +| Secret/Variable | Description | Example | +|---|---|---| +| `REGISTRY` | Registry hostname | `ghcr.io` | +| `REGISTRY_USERNAME` | Login username (or use `${{ github.actor }}`) | `myorg` | +| `REGISTRY_PASSWORD` | Login password / token | (GitHub token for GHCR) | + +For GHCR (GitHub Container Registry), `REGISTRY_PASSWORD` is `${{ secrets.GITHUB_TOKEN }}` +and no additional secret configuration is required. diff --git a/specs/001-cgc-plugin-extension/contracts/plugin-interface.md b/specs/001-cgc-plugin-extension/contracts/plugin-interface.md new file mode 100644 index 00000000..a9b71b8c --- /dev/null +++ b/specs/001-cgc-plugin-extension/contracts/plugin-interface.md @@ -0,0 +1,241 @@ +# Contract: CGC Plugin Interface + +**Version**: 1.0.0 +**Feature**: 001-cgc-plugin-extension +**Audience**: Plugin authors + +This document is the authoritative contract for building CGC-compatible plugins. +Plugins that satisfy this contract will be auto-discovered and loaded by CGC core. + +--- + +## 1. Package Structure + +A CGC plugin is a standard Python package installable via pip. It MUST follow this +structure: + +``` +cgc-plugin-/ +├── pyproject.toml # Entry point declarations (required) +└── src/ + └── cgc_plugin_/ + ├── __init__.py # PLUGIN_METADATA declaration (required) + ├── cli.py # CLI contract (required if contributing CLI commands) + └── mcp_tools.py # MCP contract (required if contributing MCP tools) +``` + +--- + +## 2. Plugin Metadata (REQUIRED) + +Every plugin MUST declare `PLUGIN_METADATA` in its package `__init__.py`: + +```python +PLUGIN_METADATA = { + "name": "my-plugin", # str, kebab-case, globally unique + "version": "0.1.0", # str, PEP-440 + "cgc_version_constraint": ">=0.3.0,<1.0", # str, PEP-440 specifier + "description": "One-line description", # str + "author": "Your Name", # str, optional +} +``` + +**Rules**: +- `name` MUST be unique across all installed plugins. Conflicts are resolved by + skipping the second plugin with a warning. +- `cgc_version_constraint` MUST be a valid PEP-440 specifier. Plugins whose constraint + does not match the installed CGC version are skipped at startup. +- All required fields MUST be present. A plugin with missing required fields is skipped. + +--- + +## 3. Entry Point Declarations + +In the plugin's `pyproject.toml`, declare entry points under one or both groups: + +```toml +[project.entry-points."cgc_cli_plugins"] +my-plugin = "cgc_plugin_myname.cli:get_plugin_commands" + +[project.entry-points."cgc_mcp_plugins"] +my-plugin = "cgc_plugin_myname.mcp_tools:get_mcp_tools" +``` + +- A plugin MAY declare CLI entry points only, MCP entry points only, or both. +- The entry point name (left of `=`) MUST match the plugin's `name` in `PLUGIN_METADATA`. + +--- + +## 4. CLI Contract + +If the plugin declares a `cgc_cli_plugins` entry point, the target function MUST have +this signature: + +```python +def get_plugin_commands() -> tuple[str, typer.Typer]: + """ + Returns a (command_group_name, typer_app) tuple. + + - command_group_name: str, kebab-case, globally unique across plugins + - typer_app: typer.Typer instance with commands registered on it + + MUST NOT: + - Have side effects (no database access, no file writes, no network calls) + - Raise unhandled exceptions (caught and logged by PluginRegistry) + - Import CGC internals at module level (use lazy imports inside handlers) + """ +``` + +**Example**: +```python +# cgc_plugin_myname/cli.py +import typer + +my_app = typer.Typer(help="My plugin commands") + +@my_app.command("hello") +def hello(): + """Say hello.""" + typer.echo("Hello from my-plugin!") + +def get_plugin_commands() -> tuple[str, typer.Typer]: + return ("my-plugin", my_app) +``` + +After installation, the user sees: `cgc my-plugin hello` + +--- + +## 5. MCP Contract + +If the plugin declares a `cgc_mcp_plugins` entry point, the target module MUST expose +two functions: + +### 5.1 get_mcp_tools() + +```python +def get_mcp_tools(server_context: dict) -> dict[str, dict]: + """ + Returns tool definitions for registration in CGC's MCP tool manifest. + + Args: + server_context: { + "db_manager": DatabaseManager, # shared graph DB connection + "version": str, # installed CGC version + } + + Returns: + dict mapping tool_name (str) → ToolDefinition (dict) + + MUST NOT: + - Register tools whose names conflict with built-in CGC tools + (conflicts are silently skipped with a warning) + - Raise unhandled exceptions + """ +``` + +### 5.2 get_mcp_handlers() + +```python +def get_mcp_handlers(server_context: dict) -> dict[str, callable]: + """ + Returns handler callables for each tool registered in get_mcp_tools(). + + Args: + server_context: same as get_mcp_tools() + + Returns: + dict mapping tool_name (str) → handler callable + + Handler callable signature: + def handler(**kwargs) -> dict: + # kwargs match the tool's inputSchema properties + # Returns a JSON-serialisable dict + """ +``` + +### 5.3 ToolDefinition Schema + +Each value in the `get_mcp_tools()` return dict MUST conform to this schema: + +```python +{ + "name": str, # MUST match the dict key + "description": str, # Human-readable description (shown in AI tool listings) + "inputSchema": { # JSON Schema draft-07 object + "type": "object", + "properties": { + "": { + "type": "string" | "integer" | "boolean" | "array" | "object", + "description": str, + # ... other JSON Schema keywords + } + }, + "required": [str, ...] # list of required property names + } +} +``` + +**Example**: +```python +# cgc_plugin_myname/mcp_tools.py + +def get_mcp_tools(server_context): + db = server_context["db_manager"] + return { + "myplugin_greet": { + "name": "myplugin_greet", + "description": "Greet by name", + "inputSchema": { + "type": "object", + "properties": { + "name": {"type": "string", "description": "Name to greet"} + }, + "required": ["name"] + } + } + } + +def get_mcp_handlers(server_context): + db = server_context["db_manager"] + def greet_handler(name: str) -> dict: + return {"greeting": f"Hello, {name}!"} + return {"myplugin_greet": greet_handler} +``` + +--- + +## 6. Naming Conventions + +To prevent conflicts in a shared namespace, plugin-registered names MUST be prefixed: + +| Artifact | Naming Rule | Example | +|---|---|---| +| CLI command group | plugin name (kebab-case) | `cgc otel ...` | +| MCP tool names | `_` | `otel_query_spans` | +| Graph node labels | PascalCase, no prefix needed | `Span`, `StackFrame` | +| Graph `source` values | `"runtime_"` or `"memory"` | `"runtime_otel"` | + +--- + +## 7. Error Handling Expectations + +CGC wraps all plugin calls. Plugins SHOULD still implement defensive error handling: + +- Handlers SHOULD catch database exceptions and return an `{"error": "..."}` dict + rather than raising exceptions, to produce clean error messages for AI agents. +- Handlers MUST be idempotent for write operations (use MERGE, not CREATE). +- Handlers MUST NOT retain state across calls beyond what the `db_manager` persists. + +--- + +## 8. Testing Requirements + +Plugin packages MUST include: + +- `tests/unit/` — unit tests for extraction/parsing logic (mocked database) +- `tests/integration/` — tests verifying the plugin registers correctly with a real + CGC server instance + +Plugin tests MUST pass with `pytest tests/unit tests/integration`. +Plugin tests SHOULD be runnable independently without CGC core installed (via mocks). diff --git a/specs/001-cgc-plugin-extension/data-model.md b/specs/001-cgc-plugin-extension/data-model.md new file mode 100644 index 00000000..3074846d --- /dev/null +++ b/specs/001-cgc-plugin-extension/data-model.md @@ -0,0 +1,320 @@ +# Data Model: CGC Plugin Extension System + +**Feature**: 001-cgc-plugin-extension +**Date**: 2026-03-14 + +This document describes both the in-memory runtime data model for the plugin system +and the new graph nodes/relationships added to the CGC graph schema by the plugins. + +--- + +## Part 1: Plugin System Runtime Model + +These entities exist at Python runtime only (not persisted to the graph). + +--- + +### PluginMetadata + +Declared by each plugin in `__init__.py::PLUGIN_METADATA`. Validated by `PluginRegistry` +at startup. + +| Field | Type | Required | Description | +|---|---|---|---| +| `name` | str | ✅ | Unique plugin identifier (kebab-case) | +| `version` | str | ✅ | Plugin version (PEP-440, e.g. `"0.1.0"`) | +| `cgc_version_constraint` | str | ✅ | PEP-440 specifier for compatible CGC versions (e.g. `">=0.3.0,<1.0"`) | +| `description` | str | ✅ | One-line human description | +| `author` | str | ❌ | Author name or team | + +**Validation rules**: +- `name` MUST be unique across all installed plugins +- `cgc_version_constraint` MUST be a valid PEP-440 specifier string +- Plugin is rejected if `cgc_version_constraint` does not match installed CGC version + +--- + +### PluginRegistration + +Runtime state of a successfully loaded plugin, held in `PluginRegistry.loaded_plugins`. + +| Field | Type | Description | +|---|---|---| +| `name` | str | Plugin name (from metadata) | +| `metadata` | PluginMetadata | Validated metadata dict | +| `cli_commands` | `list[Tuple[str, typer.Typer]]` | Registered command groups | +| `mcp_tools` | `dict[str, ToolDefinition]` | Registered MCP tool schemas | +| `mcp_handlers` | `dict[str, Callable]` | Tool name → handler function | +| `status` | `"loaded" \| "failed" \| "skipped"` | Load outcome | +| `failure_reason` | `str \| None` | Set when status is failed or skipped | + +--- + +### PluginRegistry + +Singleton held by the CGC process. Manages discovery, validation, and lifecycle. + +| Field | Type | Description | +|---|---|---| +| `loaded_plugins` | `dict[str, PluginRegistration]` | Name → registration for successfully loaded plugins | +| `failed_plugins` | `dict[str, str]` | Name → failure reason for failed/skipped plugins | + +**State transitions**: +``` +discovered → compatibility_check → [compatible] → loaded + → [incompatible] → skipped + → [import error] → failed + → [call error] → failed +``` + +--- + +### CLIPluginContract + +The callable contract each CLI plugin entry point MUST satisfy. + +```python +def get_plugin_commands() -> tuple[str, typer.Typer]: + """ + Returns: + (command_group_name, typer_app_instance) + + Raises: + Any exception → caught and logged by PluginRegistry; plugin skipped + """ +``` + +**Invariants**: +- `command_group_name` MUST be unique (CGC rejects duplicates with a warning) +- `typer_app` MUST be a `typer.Typer` instance +- Function MUST NOT have side effects beyond creating the Typer app + +--- + +### MCPPluginContract + +The callable contract each MCP plugin entry point MUST satisfy. + +```python +def get_mcp_tools(server_context: dict) -> dict[str, ToolDefinition]: + """ + Args: + server_context: { + "db_manager": DatabaseManager, + "version": str, + } + + Returns: + dict of tool_name → ToolDefinition + + Raises: + Any exception → caught and logged by PluginRegistry; plugin skipped + """ + +def get_mcp_handlers(server_context: dict) -> dict[str, Callable]: + """ + Returns: + dict of tool_name → handler_callable(**args) -> dict + """ +``` + +**ToolDefinition schema** (matches existing `tool_definitions.py` pattern): +```python +{ + "name": str, # MUST match dict key + "description": str, # Human description + "inputSchema": { # JSON Schema object + "type": "object", + "properties": { ... }, + "required": [ ... ] + } +} +``` + +--- + +## Part 2: Graph Schema Extensions + +New node labels and relationship types added by each plugin to the existing CGC graph. +All new nodes carry a `source` property identifying their origin layer. + +--- + +### OTEL Plugin Nodes + +#### Service + +Represents a named microservice observed in telemetry data. + +| Property | Type | Required | Index | Description | +|---|---|---|---|---| +| `name` | string | ✅ | UNIQUE | Service name from OTEL resource attributes | +| `version` | string | ❌ | — | Service version if reported | +| `environment` | string | ❌ | — | Environment tag (prod, staging, dev) | +| `source` | string | ✅ | — | Always `"runtime_otel"` | + +**Constraint**: `UNIQUE (s.name)` — service names are globally unique identifiers. + +--- + +#### Trace + +Represents a single distributed trace (root span + all children). + +| Property | Type | Required | Index | Description | +|---|---|---|---|---| +| `trace_id` | string | ✅ | UNIQUE | 128-bit trace ID as hex string | +| `root_span_id` | string | ✅ | — | Span ID of the root span | +| `started_at` | long | ✅ | — | Start time in Unix milliseconds | +| `duration_ms` | long | ✅ | — | Total trace duration in milliseconds | +| `source` | string | ✅ | — | Always `"runtime_otel"` | + +**Constraint**: `UNIQUE (t.trace_id)`. + +--- + +#### Span + +Represents a single operation within a trace. + +| Property | Type | Required | Index | Description | +|---|---|---|---|---| +| `span_id` | string | ✅ | UNIQUE | 64-bit span ID as hex string | +| `trace_id` | string | ✅ | INDEX | Parent trace ID (for batch queries) | +| `name` | string | ✅ | — | Span name | +| `service` | string | ✅ | — | Source service name | +| `kind` | string | ✅ | — | `SERVER`, `CLIENT`, `INTERNAL`, `PRODUCER`, `CONSUMER` | +| `class_name` | string | ❌ | INDEX | PHP: `code.namespace` attribute | +| `method_name` | string | ❌ | — | PHP: `code.function` attribute | +| `http_method` | string | ❌ | — | HTTP verb for SERVER/CLIENT spans | +| `http_route` | string | ❌ | INDEX | Route template (e.g. `/api/orders`) | +| `db_statement` | string | ❌ | — | SQL/query statement for DB spans | +| `duration_ms` | long | ✅ | — | Span duration in milliseconds | +| `status` | string | ✅ | — | `OK`, `ERROR`, `UNSET` | +| `source` | string | ✅ | — | Always `"runtime_otel"` | + +**Constraints**: `UNIQUE (s.span_id)`. +**Indexes**: `(s.trace_id)`, `(s.class_name)`, `(s.http_route)`. + +--- + +### Xdebug Plugin Nodes + +#### StackFrame + +Represents a single frame in a PHP execution call stack captured via DBGp. + +| Property | Type | Required | Index | Description | +|---|---|---|---|---| +| `frame_id` | string | ✅ | UNIQUE | Hash of `class_name::method_name:file_path:line` | +| `class_name` | string | ✅ | INDEX | PHP class name (fully qualified) | +| `method_name` | string | ✅ | — | PHP method name | +| `fqn` | string | ✅ | INDEX | `ClassName::methodName` for correlation | +| `file_path` | string | ✅ | — | Absolute file path from DBGp | +| `line` | int | ✅ | — | Line number | +| `depth` | int | ✅ | — | Call stack depth (0 = top) | +| `chain_hash` | string | ✅ | INDEX | Deduplication hash of the full call chain | +| `observation_count` | int | ✅ | — | Number of times this chain was observed | +| `source` | string | ✅ | — | Always `"runtime_xdebug"` | + +**Constraint**: `UNIQUE (sf.frame_id)`. +**Index**: `(sf.fqn)` for `RESOLVES_TO` correlation lookups. + +--- + +### Memory Plugin Nodes + +#### Memory + +Represents a structured knowledge entity (spec, decision, research, bug, feature). +Provided by the `mcp/neo4j-memory` service; schema documented here for cross-layer +query reference. + +| Property | Type | Required | Index | Description | +|---|---|---|---|---| +| `id` | string | ✅ | UNIQUE | UUID | +| `name` | string | ✅ | FULLTEXT | Human-readable entity name | +| `entity_type` | string | ✅ | FULLTEXT | `spec`, `decision`, `research`, `bug`, `feature`, `integration` | +| `created_at` | datetime | ✅ | — | Creation timestamp | +| `updated_at` | datetime | ✅ | — | Last update timestamp | +| `source` | string | ✅ | — | Always `"memory"` | + +--- + +#### Observation + +A single piece of content attached to a Memory entity. + +| Property | Type | Required | Index | Description | +|---|---|---|---|---| +| `content` | string | ✅ | FULLTEXT | The observation text | +| `created_at` | datetime | ✅ | — | Creation timestamp | + +--- + +## Part 3: Graph Relationship Extensions + +New relationships added by the plugins. Existing CGC relationships are not modified. + +--- + +### OTEL Relationships + +| Relationship | From → To | Properties | Description | +|---|---|---|---| +| `CHILD_OF` | Span → Span | — | Parent-child span hierarchy | +| `PART_OF` | Span → Trace | — | Span belongs to trace | +| `ORIGINATED_FROM` | Trace → Service | — | Trace started in service | +| `CALLS_SERVICE` | Span → Service | — | Cross-service call (CLIENT spans only) | +| `CORRELATES_TO` | Span → Method | `confidence: "fqn_match"` | Runtime → static correlation | + +--- + +### Xdebug Relationships + +| Relationship | From → To | Properties | Description | +|---|---|---|---| +| `CALLED_BY` | StackFrame → StackFrame | `depth_diff: int` | Call chain (child called by parent) | +| `RESOLVES_TO` | StackFrame → Method | `match_type: "fqn_exact"` | Frame → static method node | + +--- + +### Memory Relationships + +| Relationship | From → To | Properties | Description | +|---|---|---|---| +| `HAS_OBSERVATION` | Memory → Observation | — | Knowledge entity has content | +| `RELATES_TO` | Memory → Memory | `relation: string` | Inter-entity links | +| `DESCRIBES` | Memory → Class | — | Knowledge about a class | +| `DESCRIBES` | Memory → Method | — | Knowledge about a method | +| `COVERS` | Memory → Span | — | Knowledge about a runtime operation | + +--- + +## Part 4: Schema Migration + +All new node labels and relationship types are additive — they do not modify existing +CGC node labels (`File`, `Class`, `Method`, `Function`) or existing relationships +(`CALLS`, `IMPORTS`, `INHERITS`, `DEFINES`). + +Required Cypher initialization statements (added to `config/neo4j/init.cypher`): + +```cypher +-- OTEL constraints & indexes +CREATE CONSTRAINT service_name IF NOT EXISTS FOR (s:Service) REQUIRE s.name IS UNIQUE; +CREATE CONSTRAINT trace_id IF NOT EXISTS FOR (t:Trace) REQUIRE t.trace_id IS UNIQUE; +CREATE CONSTRAINT span_id IF NOT EXISTS FOR (s:Span) REQUIRE s.span_id IS UNIQUE; +CREATE INDEX span_trace IF NOT EXISTS FOR (s:Span) ON (s.trace_id); +CREATE INDEX span_class IF NOT EXISTS FOR (s:Span) ON (s.class_name); +CREATE INDEX span_route IF NOT EXISTS FOR (s:Span) ON (s.http_route); + +-- Xdebug constraints & indexes +CREATE CONSTRAINT frame_id IF NOT EXISTS FOR (sf:StackFrame) REQUIRE sf.frame_id IS UNIQUE; +CREATE INDEX frame_fqn IF NOT EXISTS FOR (sf:StackFrame) ON (sf.fqn); + +-- Memory full-text indexes (managed by mcp/neo4j-memory service) +CREATE FULLTEXT INDEX memory_search IF NOT EXISTS + FOR (m:Memory) ON EACH [m.name, m.entity_type]; +CREATE FULLTEXT INDEX observation_search IF NOT EXISTS + FOR (o:Observation) ON EACH [o.content]; +``` diff --git a/specs/001-cgc-plugin-extension/plan.md b/specs/001-cgc-plugin-extension/plan.md new file mode 100644 index 00000000..3ffd3f1b --- /dev/null +++ b/specs/001-cgc-plugin-extension/plan.md @@ -0,0 +1,176 @@ +# Implementation Plan: CGC Plugin Extension System + +**Branch**: `001-cgc-plugin-extension` | **Date**: 2026-03-14 | **Spec**: [spec.md](spec.md) +**Input**: Feature specification from `specs/001-cgc-plugin-extension/spec.md` + +## Summary + +Extend CodeGraphContext with a Python entry-points plugin system that allows independently +installable packages to contribute CLI commands (Typer) and MCP tools without modifying +CGC core. Three first-party plugins ship with the extension: an OTEL span processor (runtime +intelligence), an Xdebug DBGp listener (dev-time stack traces), and a memory knowledge +wrapper (project context). A shared GitHub Actions matrix CI/CD pipeline builds and publishes +versioned Docker images for each plugin service. All plugin data flows into the existing +Neo4j/FalkorDB graph, enabling cross-layer queries across static code, runtime execution, +and project knowledge. + +## Technical Context + +**Language/Version**: Python 3.10+ (constitutional constraint) +**Primary Dependencies**: +- Plugin system: `importlib.metadata` (stdlib), `packaging>=23.0` (version constraint checking) +- OTEL plugin: `grpcio>=1.57.0`, `opentelemetry-proto>=0.43b0`, `opentelemetry-sdk>=1.20.0` +- Xdebug plugin: stdlib only (`socket`, `xml.etree.ElementTree`, `hashlib`) +- Memory plugin: wraps `mcp/neo4j-memory` Docker image; thin Python package only +- All plugins: `typer[all]>=0.9.0`, `neo4j>=5.15.0` (shared with core) + +**Storage**: Neo4j (production) / FalkorDB (default) — same shared instance as CGC core; +new additive node labels and relationships per `data-model.md` + +**Testing**: pytest + pytest-asyncio; existing `tests/run_tests.sh` extended with +`tests/unit/plugin/`, `tests/integration/plugin/`, `tests/e2e/plugin/` + +**Target Platform**: Linux server (Docker containers); Kubernetes compatible (no host +networking, env-var-only config) + +**Project Type**: Python library + CLI extensions + containerised microservices + +**Performance Goals**: +- CGC startup with all 3 plugins: ≤ 15 seconds +- Span data queryable within 10 seconds of request completion under normal load +- Plugin load failure: ≤ 5-second timeout per plugin (SIGALRM) + +**Constraints**: +- Plugin failures MUST NOT crash CGC core (strict isolation) +- No credentials baked into container images +- `./tests/run_tests.sh fast` MUST pass after each phase +- Xdebug plugin MUST default to disabled (security: TCP listener) + +**Scale/Scope**: 3 plugin packages, 1 shared CI/CD pipeline, 5 container services + +## Constitution Check + +*GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.* + +| Principle | Status | Evidence | +|---|---|---| +| **I. Graph-First Architecture** | ✅ PASS | All plugin output (spans, stack frames, memory entities) writes to the graph as typed nodes + relationships per `data-model.md`. No flat data structures. Graph schema is the output target for all three plugins. | +| **II. Dual Interface — CLI + MCP** | ✅ PASS | Each plugin MUST contribute both CLI commands AND MCP tools (per plugin interface contract). The plugin contract enforces parity by design. | +| **III. Testing Pyramid** | ✅ PASS | Plugin packages include `tests/unit/` and `tests/integration/`. `./tests/run_tests.sh fast` is extended to cover plugin directories. E2E tests cover the full plugin lifecycle. Tests written and observed to FAIL before implementation (Red-Green-Refactor). | +| **IV. Multi-Language Parser Parity** | ✅ PASS | No new language parsers introduced. Runtime nodes carry `source` property (`"runtime_otel"`, `"runtime_xdebug"`, `"memory"`) that distinguish origin layers without breaking existing cross-language queries. | +| **V. Simplicity** | ⚠️ JUSTIFIED | Plugin registry is an abstraction. Justified because: (a) the feature requires extensibility without forking core — a non-negotiable requirement; (b) `importlib.metadata` entry-points is Python stdlib — minimal abstraction; (c) without a registry, adding each plugin would require modifying `server.py` and `cli/main.py` permanently, producing a worse monolith. See Complexity Tracking below. | + +*Post-Phase 1 re-check*: ✅ Design satisfies all five principles. No new violations introduced. + +## Project Structure + +### Documentation (this feature) + +```text +specs/001-cgc-plugin-extension/ +├── plan.md # This file +├── research.md # Phase 0 output +├── data-model.md # Phase 1 output +├── quickstart.md # Phase 1 output +├── contracts/ +│ ├── plugin-interface.md # Plugin author contract +│ └── cicd-pipeline.md # CI/CD service registration contract +└── tasks.md # Phase 2 output (/speckit.tasks command) +``` + +### Source Code (repository root) + +```text +# Core CGC modifications (existing package) +src/codegraphcontext/ +├── plugin_registry.py # NEW: PluginRegistry class, isolation wrappers +├── cli/ +│ └── main.py # MODIFIED: call load_plugin_cli_commands() at startup +└── server.py # MODIFIED: call _load_plugin_tools() in __init__ + +# New plugin packages +plugins/ +├── cgc-plugin-otel/ +│ ├── pyproject.toml +│ ├── Dockerfile +│ └── src/cgc_plugin_otel/ +│ ├── __init__.py # PLUGIN_METADATA +│ ├── cli.py # get_plugin_commands() → ("otel", typer.Typer) +│ ├── mcp_tools.py # get_mcp_tools(), get_mcp_handlers() +│ ├── receiver.py # gRPC OTLP receiver (grpcio + opentelemetry-proto) +│ ├── span_processor.py # PHP attribute extraction + correlation logic +│ └── neo4j_writer.py # Async batch writer with dead-letter queue +│ +├── cgc-plugin-xdebug/ +│ ├── pyproject.toml +│ ├── Dockerfile +│ └── src/cgc_plugin_xdebug/ +│ ├── __init__.py # PLUGIN_METADATA +│ ├── cli.py # get_plugin_commands() → ("xdebug", typer.Typer) +│ ├── mcp_tools.py # get_mcp_tools(), get_mcp_handlers() +│ ├── dbgp_server.py # TCP DBGp listener + XML stack frame parser +│ └── neo4j_writer.py # Frame upsert + CALLED_BY chain + deduplication +│ +└── cgc-plugin-memory/ + ├── pyproject.toml + ├── Dockerfile # Wraps mcp/neo4j-memory + proxy layer + └── src/cgc_plugin_memory/ + ├── __init__.py # PLUGIN_METADATA + ├── cli.py # get_plugin_commands() → ("memory", typer.Typer) + └── mcp_tools.py # get_mcp_tools(), get_mcp_handlers() (proxy) + +# Tests (additions to existing structure) +tests/ +├── unit/ +│ └── plugin/ +│ ├── test_plugin_registry.py # PluginRegistry unit tests (mocked) +│ ├── test_otel_processor.py # Span extraction logic +│ └── test_xdebug_parser.py # DBGp XML parsing + deduplication +├── integration/ +│ └── plugin/ +│ ├── test_plugin_load.py # Plugin discovery + load integration +│ ├── test_otel_integration.py # OTLP receive → graph write +│ └── test_memory_integration.py # Memory store → graph node +└── e2e/ + └── plugin/ + └── test_plugin_lifecycle.py # Full install/use/uninstall user journey + +# CI/CD +.github/ +├── services.json # NEW: service list for Docker matrix +└── workflows/ + ├── docker-publish.yml # MODIFIED: matrix over services.json + └── test-plugins.yml # NEW: per-plugin fast test suite + +# Deployment +docker-compose.yml # MODIFIED: add otel + memory services +docker-compose.dev.yml # MODIFIED: add xdebug service +config/ +├── otel-collector/ +│ └── config.yaml # NEW: OTel Collector pipeline config +└── neo4j/ + └── init.cypher # MODIFIED: add plugin schema constraints + +k8s/ +├── cgc-plugin-otel/ +│ ├── deployment.yaml +│ └── service.yaml +└── cgc-plugin-memory/ + ├── deployment.yaml + └── service.yaml +``` + +**Structure Decision**: Multi-package layout under `plugins/` with independent +`pyproject.toml` per plugin. This matches the research recommendation (R-010) and is the +standard Python ecosystem pattern for monorepo plugin families. Plugin packages are +installable independently (`pip install codegraphcontext[otel]`) or via optional extras +in the root `pyproject.toml`. Each plugin that exposes a container service has its own +`Dockerfile` in the plugin directory. + +## Complexity Tracking + +| Violation | Why Needed | Simpler Alternative Rejected Because | +|---|---|---| +| Plugin registry abstraction | Feature explicitly requires extensibility without forking core. Three current plugins + third-party extensibility require a clean registration boundary. | Hardcoding plugins in `server.py`/`main.py` defeats the extensibility requirement entirely. There is no simpler path to the stated goal. | +| gRPC server in OTEL plugin | OTLP protocol uses gRPC. The Python opentelemetry-sdk is tracer-side only and cannot act as a receiver. | Pure HTTP OTLP would require the same gRPC-level effort and provides less tooling ecosystem support. The OTel Collector (sidecar) already handles the edge; gRPC is the right interface for collector → processor. | +| Multiple new graph node types | Runtime and memory layers produce genuinely different data (spans, frames, knowledge entities). Reusing existing `Method`/`Class` nodes for runtime data would corrupt the static layer. | Cannot collapse runtime nodes into static nodes — they represent different semantic things (observed execution vs. declared code). The `source` property differentiates them without schema explosion. | diff --git a/specs/001-cgc-plugin-extension/quickstart.md b/specs/001-cgc-plugin-extension/quickstart.md new file mode 100644 index 00000000..ab880bdc --- /dev/null +++ b/specs/001-cgc-plugin-extension/quickstart.md @@ -0,0 +1,211 @@ +# Quickstart: CGC Plugin Extension System + +**Feature**: 001-cgc-plugin-extension +**Audience**: Developers setting up CGC-X locally and contributors building plugins + +--- + +## Prerequisites + +- Python 3.10+ +- pip / virtualenv +- Docker + Docker Compose (for container services) +- A running Neo4j instance (or use the provided docker-compose) + +--- + +## 1. Run the Full CGC-X Stack (Docker Compose) + +The fastest way to get the full stack running: + +```bash +# Clone the repo +git clone https://github.com/CodeGraphContext/CodeGraphContext +cd CodeGraphContext + +# Copy and configure environment +cp .env.example .env +# Edit .env: set NEO4J_PASSWORD and DOMAIN + +# Start core + memory plugin (production profile) +docker compose up -d + +# Start with Xdebug listener (dev profile — adds xdebug service) +docker compose -f docker-compose.yml -f docker-compose.dev.yml up -d +``` + +**Services started**: +| Service | URL / Port | Purpose | +|---|---|---| +| Neo4j | bolt://localhost:7687 | Shared graph database | +| CGC core | MCP at localhost:8080 | Static code indexing | +| OTEL plugin | gRPC at localhost:5317 | Runtime span ingestion | +| Memory plugin | MCP at localhost:8766 | Project knowledge storage | +| Xdebug listener (dev) | TCP at localhost:9003 | Dev-time stack traces | + +--- + +## 2. Install CGC with Plugins (Python — Development Mode) + +For local development or when running without Docker: + +```bash +# Create a virtual environment +python -m venv .venv +source .venv/bin/activate + +# Install CGC core + all plugins in editable mode +pip install -e . +pip install -e plugins/cgc-plugin-otel +pip install -e plugins/cgc-plugin-xdebug +pip install -e plugins/cgc-plugin-memory + +# Verify plugins loaded +cgc --help +# Should show: otel, xdebug, memory command groups alongside built-in commands +``` + +**Install specific plugins only** (production use): +```bash +pip install codegraphcontext[otel] # core + OTEL plugin +pip install codegraphcontext[memory] # core + memory plugin +pip install codegraphcontext[all] # core + all plugins +``` + +--- + +## 3. Verify Plugin Discovery + +```bash +# List all loaded plugins +cgc plugin list + +# Expected output: +# ✓ cgc-plugin-otel v0.1.0 3 tools (otel_query_spans, otel_list_services, otel_cross_layer_query) 3 commands +# ✓ cgc-plugin-memory v0.1.0 4 tools (memory_store, memory_search, memory_undocumented, memory_link) 4 commands +# ✓ cgc-plugin-xdebug v0.1.0 2 tools (xdebug_list_chains, xdebug_query_chain) 3 commands (dev only) +``` + +--- + +## 4. Index a Repository + +```bash +# Index a local PHP/Laravel project +cgc index /path/to/your/laravel-project + +# Verify nodes were created +cgc query "MATCH (c:Class) RETURN c.name LIMIT 5" +``` + +--- + +## 5. Enable Runtime Intelligence (OTEL Plugin) + +Add to your Laravel application's `.env`: +```ini +OTEL_PHP_AUTOLOAD_ENABLED=true +OTEL_SERVICE_NAME=my-service +OTEL_TRACES_EXPORTER=otlp +OTEL_EXPORTER_OTLP_PROTOCOL=grpc +OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4317 +``` + +Send a request to your application. Verify spans appear in the graph: +```bash +cgc otel query-spans --route /api/orders --limit 5 +``` + +Or via MCP tool: +```json +{ + "tool": "otel_query_spans", + "arguments": {"http_route": "/api/orders", "limit": 5} +} +``` + +--- + +## 6. Store Project Knowledge (Memory Plugin) + +```bash +# Store a spec for a class +cgc memory store \ + --type spec \ + --name "OrderController spec" \ + --content "Handles order creation and status transitions" \ + --links-to "App\\Http\\Controllers\\OrderController" + +# Query: which code has no spec? +cgc memory undocumented +``` + +--- + +## 7. Enable Dev-Time Traces (Xdebug Plugin) + +Ensure your PHP application has Xdebug installed with these settings: +```ini +xdebug.mode=debug,trace +xdebug.client_host=localhost ; or Docker host IP +xdebug.client_port=9003 +xdebug.start_with_request=trigger +``` + +Trigger a trace by setting the `XDEBUG_TRIGGER` cookie in your browser, then query: +```bash +cgc xdebug list-chains --limit 10 +``` + +--- + +## 8. Cross-Layer Query Example + +After indexing code + collecting runtime spans + storing specs, run this cross-layer +query to find running code with no specification: + +```bash +cgc query " +MATCH (m:Method)<-[:CORRELATES_TO]-(s:Span) +WHERE NOT EXISTS { MATCH (mem:Memory)-[:DESCRIBES]->(m) } +RETURN m.fqn, count(s) AS executions +ORDER BY executions DESC +LIMIT 20 +" +``` + +--- + +## 9. Build and Push Container Images + +```bash +# Trigger a release build (creates all plugin images) +git tag v0.1.0 +git push origin v0.1.0 +# GitHub Actions automatically builds and pushes: +# ghcr.io//cgc-core:0.1.0 +# ghcr.io//cgc-plugin-otel:0.1.0 +# ghcr.io//cgc-plugin-memory:0.1.0 + +# Monitor at: github.com//CodeGraphContext/actions +``` + +--- + +## 10. Write Your Own Plugin + +```bash +# Use the plugin scaffold (coming in a future task) +# For now, copy the example plugin: +cp -r plugins/cgc-plugin-memory plugins/cgc-plugin-myname + +# Edit pyproject.toml: change name, entry points, dependencies +# Edit src/cgc_plugin_myname/__init__.py: update PLUGIN_METADATA +# Implement cli.py and mcp_tools.py following the plugin-interface.md contract +# Install and test: +pip install -e plugins/cgc-plugin-myname +cgc plugin list # Should show your plugin +``` + +See `specs/001-cgc-plugin-extension/contracts/plugin-interface.md` for the full +plugin contract specification. diff --git a/specs/001-cgc-plugin-extension/research.md b/specs/001-cgc-plugin-extension/research.md new file mode 100644 index 00000000..879b35be --- /dev/null +++ b/specs/001-cgc-plugin-extension/research.md @@ -0,0 +1,266 @@ +# Research: CGC Plugin Extension System + +**Feature**: 001-cgc-plugin-extension +**Date**: 2026-03-14 +**Status**: Complete — all NEEDS CLARIFICATION resolved + +--- + +## R-001: Plugin Discovery Mechanism + +**Decision**: Use Python `importlib.metadata.entry_points()` (stdlib, Python 3.10+) with +two named groups: `cgc_cli_plugins` and `cgc_mcp_plugins`. + +**Rationale**: Entry points are the Python ecosystem's standard plugin discovery contract. +They require zero runtime overhead beyond package installation — no config files, no +manual registration, no import scanning. Every tool in the Python ecosystem (pytest, +flask, flake8) uses this pattern. It is stdlib in Python 3.10+ (no extra dependency). + +**How it works**: +- Plugin packages declare entry points in their own `pyproject.toml` +- `pip install` indexes entry point metadata into the environment +- CGC calls `entry_points(group="cgc_cli_plugins")` at startup to discover all installed + plugins that contribute CLI commands +- CGC calls `entry_points(group="cgc_mcp_plugins")` to discover MCP tool contributors +- Each group resolves to a callable that CGC invokes to receive the plugin's registration + +**Alternatives considered**: +- Filesystem scanning (explicit plugin dir) — more brittle, non-standard, breaks with + virtual environments +- Config file listing plugins — requires manual edits (violates FR-002 "zero edits") +- Import path hooks — too low-level, fragile, hard to debug + +--- + +## R-002: CLI Plugin Interface + +**Decision**: Each CLI plugin entry point resolves to a function +`get_plugin_commands() -> Tuple[str, typer.Typer]` that returns a +`(command_group_name, typer_app_instance)` tuple. CGC calls +`app.add_typer(plugin_app, name=cmd_name)` for each loaded plugin. + +**Rationale**: Typer's `add_typer()` is the idiomatic way to compose command groups. The +pattern requires the plugin to own its Typer app (clean separation), and CGC to own the +top-level `app` (clean host). Returning a tuple rather than a dict is simpler for the +common case (one command group per plugin) and is consistently typed. + +**Startup sequence**: +``` +CLI main.py imports → PluginRegistry discovers cgc_cli_plugins entries → +calls each get_plugin_commands() → app.add_typer() for each → Typer starts +``` + +**Alternatives considered**: +- Plugin directly calls `app.add_typer()` — creates bidirectional coupling; plugin + imports core at registration time which can cause circular imports +- Plugin returns a Click group — Typer wraps Click but mixing levels is error-prone; + Typer's add_typer is cleaner + +--- + +## R-003: MCP Plugin Interface + +**Decision**: Each MCP plugin entry point resolves to a function +`get_mcp_tools(server_context: dict) -> dict[str, ToolDefinition]` that returns a +mapping of tool name → tool definition dict (same schema as core `tool_definitions.py`). +CGC's `MCPServer._load_plugin_tools()` merges these into its tools manifest and routes +calls via a unified `handle_tool_call()` dispatcher. + +**Server context passed to plugins** (minimal, read-only intent): +```python +{ + "db_manager": self.db_manager, # shared database connection + "version": "x.y.z", # CGC core version string +} +``` + +**Rationale**: Plugins receive the `db_manager` so they can share the existing database +connection rather than opening independent connections (violating the constitution's +single-database principle). Passing only what is needed (not `self`) prevents plugins +from calling internal server methods they shouldn't access. + +**Tool handler registration**: The plugin's `get_mcp_tools()` return value maps +tool names to JSON Schema definitions. The plugin ALSO registers handler callables +in a separate `get_mcp_handlers()` function (or combined in a single object). The +server stores handlers in `self.plugin_tool_handlers` dict and routes calls there +before checking built-in handlers. + +**Alternatives considered**: +- Subclass MCPServer per plugin — couples plugin to server implementation; not viable + for third-party plugins +- Plugin monkey-patches server — completely unsafe and untestable +- gRPC plugin protocol — overkill for in-process plugins; entry-points are sufficient + +--- + +## R-004: Plugin Version Compatibility + +**Decision**: Each plugin's `__init__.py` declares `PLUGIN_METADATA` dict with a +`cgc_version_constraint` key using PEP-440 version specifier syntax (e.g. +`">=0.3.0,<1.0"`). CGC's `PluginRegistry` validates this against the installed +`codegraphcontext` package version using `packaging.specifiers.SpecifierSet`. + +**On mismatch**: plugin is skipped with a WARNING log; all compatible plugins still load; +no error is raised to the user unless zero plugins load. + +**Rationale**: `packaging` is already an indirect dependency of pip and is present in +all virtual environments. PEP-440 specifiers are the Python standard for version +constraints. Soft-fail (warn, skip) rather than hard-fail ensures partial plugin +ecosystems remain usable. + +**Alternatives considered**: +- Semver-only checking — PEP-440 is a superset and already the ecosystem standard +- No version checking — risks silent breakage when core APIs change + +--- + +## R-005: Plugin Isolation (Error Containment) + +**Decision**: Wrap each plugin load in a `try/except Exception` block. Use a +`PluginRegistry` class that catches `ImportError`, `AttributeError`, `TimeoutError`, and +generic `Exception` at each stage (import, metadata read, command/tool registration). +A broken plugin logs an error and sets `failed_plugins[name] = reason`; it NEVER +propagates an exception to the host process. + +**Timeout**: On Unix, `signal.SIGALRM` with a 5-second timeout prevents hanging imports. +(Windows lacks SIGALRM — on Windows, timeout is skipped with a warning.) + +**Startup summary**: After all plugins are processed, CGC logs: +``` +CGC started with 19 built-in tools and 6 plugin tools (1 plugin failed). + ✓ cgc-plugin-otel 4 tools + ✓ cgc-plugin-memory 2 tools + ✗ cgc-plugin-xdebug SKIPPED: missing dependency 'dbgp' +``` + +**Rationale**: The spec requires (FR-003) that plugin failures do not prevent CGC core +from starting. Isolation at the `PluginRegistry` boundary is the cleanest enforcement. + +--- + +## R-006: OTEL Span Receiver Architecture + +**Decision**: Deploy the standard OpenTelemetry Collector (`otel/opentelemetry-collector-contrib`) +as a sidecar. It receives OTLP from applications and forwards to the OTEL plugin service +via OTLP gRPC. The plugin service implements a Python gRPC server using `grpcio` + +`opentelemetry-proto` protobuf definitions. + +**Rationale**: The Python `opentelemetry-sdk` is tracer-side only — it cannot act as an +OTLP gRPC receiver endpoint. Using the official OTel Collector as a sidecar provides +batching, retry, filtering, and sampling for free before spans reach the custom processor. +This is the established production pattern (used by Datadog, Honeycomb, Jaeger agents). + +**Key packages**: +- `grpcio>=1.57.0` — gRPC server implementation +- `opentelemetry-proto>=0.43b0` — generated protobuf/gRPC classes +- `neo4j>=5.15.0` — async Python driver + +**Write pattern**: Async batch writer using `asyncio.Queue` with configurable +`max_batch_size=100` and `max_wait_ms=5000`. MERGE on `(trace_id, span_id)` for +idempotency. Dead-letter queue for resilience when Neo4j is temporarily unavailable. + +**Alternatives considered**: +- Pure Python OTLP HTTP receiver (no gRPC) — simpler but less efficient; OTel Collector + already handles the gRPC ↔ HTTP translation if needed +- Direct OTLP from app → Python service — fragile; the Collector adds resilience + +--- + +## R-007: Xdebug DBGp Listener + +**Decision**: Implement a minimal TCP server using Python's stdlib `socket` and +`xml.etree.ElementTree` modules. No external DBGp library is required — the protocol +is XML over TCP and is simple enough to implement directly. + +**Protocol flow**: PHP Xdebug connects → init packet received → `run` command sent → +on each breakpoint, send `stack_get` → parse XML response → upsert `StackFrame` nodes +and `CALLED_BY` edges → send `run` → repeat until connection closes. + +**Deduplication**: Hash the call chain (`sha256(class::method|...) [:16]`) with an +LRU cache (size configurable, default 10,000). If hash seen recently, skip upsert. +This prevents the same execution path from creating duplicate graph structure. + +**Dev-only deployment**: The Xdebug plugin starts its TCP listener only when enabled +(`CGC_PLUGIN_XDEBUG_ENABLED=true`). In production Docker Compose, the `xdebug` service +is absent from the default compose file; it exists only in `docker-compose.dev.yml`. + +**Rationale**: Xdebug is a dev tool. Running a DBGp listener in production is a security +risk. The plugin MUST default to disabled and require explicit opt-in. + +--- + +## R-008: Memory Plugin Architecture + +**Decision**: The memory plugin is a thin wrapper. The underlying storage is provided by +the `mcp/neo4j-memory` Docker image (official, maintained). The plugin package in CGC: +1. Provides a `cgc plugin memory enable/disable/status` CLI command group +2. Proxies MCP tool definitions so they appear in CGC's tool listing even though the + actual service runs separately +3. Provides a `docker-compose.yml` snippet and Kubernetes manifests for deployment + +**Rationale**: The research document explicitly states "Why mcp/neo4j-memory rather than +a custom memory service? It's maintained, well-documented, and covers the generic memory +use case well." Building custom memory storage would violate the Simplicity principle +(V) — unnecessary complexity for solved problems. + +**Neo4j sharing**: The memory plugin connects to the same Neo4j instance as CGC core, +enabling cross-layer queries (`(Memory)-[:DESCRIBES]->(Method)`) as specified. + +--- + +## R-009: CI/CD Pipeline Architecture + +**Decision**: GitHub Actions matrix strategy with `fail-fast: false`. Services defined +in `.github/services.json` as a JSON array. Shared logic for checkout, Docker login, +version tag extraction, and metadata is in the matrix job — matrix jobs inherit shared +steps via `needs` dependencies from a `setup` job that outputs the services matrix. + +**Tagging strategy**: `docker/metadata-action@v5` generates: +- Semver tags from git tags (`v1.2.3` → `1.2.3`, `1.2`, `1`) +- `latest` on default branch pushes +- Branch name tags for non-default branches + +**Health check**: After `docker/build-push-action@v5` with `push: false` (local build), +load image and run a service-specific smoke test. Only push if smoke test passes. + +**Adding a new service**: Add one JSON object to `.github/services.json`. Zero workflow +logic changes. + +**Key action versions** (current as of research date): +- `docker/setup-buildx-action@v3` +- `docker/build-push-action@v5` +- `docker/login-action@v3` +- `docker/metadata-action@v5` + +**Alternatives considered**: +- One workflow file per service — massive duplication, violates FR-030 +- Reusable workflow (workflow_call) — more complex than needed; matrix is sufficient + +--- + +## R-010: Monorepo Package Layout + +**Decision**: Plugin packages live in `plugins/` subdirectory, each as an independently +installable Python package with its own `pyproject.toml`. Plugin services that run as +standalone containers (OTEL, Xdebug) also have a `Dockerfile` in their directory. The +memory plugin's "service" is the third-party `mcp/neo4j-memory` image; its plugin +directory contains only the Python package code and deployment manifests. + +**Development installation**: +```bash +pip install -e . # CGC core +pip install -e plugins/cgc-plugin-otel +pip install -e plugins/cgc-plugin-xdebug +pip install -e plugins/cgc-plugin-memory +``` + +After this, `cgc --help` shows plugin commands automatically. + +**Production installation** (users who want only specific plugins): +```bash +pip install codegraphcontext # Core only +pip install codegraphcontext[otel] # Core + OTEL plugin (via extras) +pip install codegraphcontext[memory] # Core + memory plugin +``` + +This is achieved by declaring plugins as optional extras in the root `pyproject.toml`. diff --git a/specs/001-cgc-plugin-extension/spec.md b/specs/001-cgc-plugin-extension/spec.md new file mode 100644 index 00000000..5aceb21b --- /dev/null +++ b/specs/001-cgc-plugin-extension/spec.md @@ -0,0 +1,362 @@ +# Feature Specification: CGC Plugin Extension System + +**Feature Branch**: `001-cgc-plugin-extension` +**Created**: 2026-03-14 +**Status**: Draft +**Input**: Based on research in `cgc-extended-spec.md` — extend CGC to support runtime +memory and project knowledge layers via a plugin/addon pattern for CLI and MCP, with a +common CI/CD pipeline for Docker/K8s images. + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - Plugin Extensibility Foundation (Priority: P1) + +A CGC contributor or third-party developer wants to extend CGC with new capabilities +(new data sources, new MCP tools, new CLI commands) without modifying the core CGC +codebase. They build a self-contained addon package that declares its CLI commands and +MCP tools, publishes it separately, and CGC discovers and loads it automatically when +installed. + +**Why this priority**: All other stories depend on a functioning plugin system. Without +the foundation, the runtime, memory, and CI/CD stories cannot be independently developed +or released. This is the architectural backbone that makes the project composable. + +**Independent Test**: Install CGC core alone and verify it starts correctly. Then install +a minimal stub plugin; verify CGC discovers the plugin, the plugin's CLI command appears +in `cgc --help`, and its MCP tool appears in the MCP tool listing — without any changes +to core CGC code. + +**Acceptance Scenarios**: + +1. **Given** CGC core is installed without any plugins, **When** a user runs the CGC + CLI, **Then** only built-in core commands appear and no plugin-related errors occur. +2. **Given** a plugin package is installed in the same environment, **When** CGC starts, + **Then** the plugin's CLI commands and MCP tools are automatically available alongside + core capabilities. +3. **Given** a plugin is installed, **When** the plugin is uninstalled, **Then** CGC + starts cleanly without the plugin's commands or tools and without crashing. +4. **Given** two plugins are installed simultaneously, **When** CGC starts, **Then** + both plugins' commands and tools are available with no naming conflicts for distinct + plugins. +5. **Given** a plugin declares an incompatible version constraint, **When** CGC loads + plugins, **Then** the incompatible plugin is skipped with a clear warning stating the + version mismatch, and all compatible plugins still load. + +--- + +### User Story 2 - Runtime Intelligence via OTEL Plugin (Priority: P2) + +A backend developer running a PHP/Laravel application wants to understand what code +actually executes at runtime, not just what the static graph shows. They enable the +OTEL plugin, point their application's telemetry at the CGC OTEL endpoint, and can then +ask their AI assistant questions that combine runtime call data with static code +structure — for example, "which methods were called in the last hour that have no test +coverage" or "show the full execution path for a POST /api/orders request." + +**Why this priority**: The OTEL plugin is the highest-value runtime layer. It is +non-invasive (standard OTEL instrumentation already used in many projects), production- +safe, and delivers cross-layer queries immediately once spans flow into the graph. + +**Independent Test**: Start CGC with the OTEL plugin enabled. Send a sample trace (or +synthetic span payload) to the plugin's ingestion endpoint. Verify that the graph now +contains runtime nodes linked to static code nodes from a pre-indexed repository, and +that a cross-layer query returns meaningful results. + +**Acceptance Scenarios**: + +1. **Given** the OTEL plugin is enabled and a repository is indexed, **When** a + telemetry-instrumented application sends request traces, **Then** runtime call data + appears in the graph within 10 seconds of the request completing. +2. **Given** runtime nodes exist in the graph, **When** an AI assistant queries + "which methods ran during request X", **Then** the MCP tool returns a linked result + showing both the runtime call chain and the corresponding static code nodes. +3. **Given** a cross-service call occurs (service A calls service B), **When** spans + from both services are received, **Then** the graph contains an edge connecting the + two services and the call is queryable as a single path. +4. **Given** health-check or noise spans are received, **When** ingestion runs, **Then** + noise spans are filtered out and do not pollute the graph. +5. **Given** the OTEL plugin is disabled or not installed, **When** CGC starts, **Then** + no OTEL-related commands or tools appear and the core graph is unaffected. + +--- + +### User Story 3 - Development Traces via Xdebug Plugin (Priority: P3) + +A PHP developer debugging a complex feature wants method-level execution traces that +capture exactly which concrete class implementations ran (not just the interface), which +is information OTEL spans don't always provide. They enable the Xdebug listener plugin +in their development environment and selectively trigger traces for specific requests. +The resulting call-chain graph is linked back to CGC's static code nodes so they can +navigate from "what ran" to "where it's defined." + +**Why this priority**: This plugin is development/staging-only and requires Xdebug on +the target application, limiting its audience. It delivers deep, precise traces but is +not needed in production. It depends on the plugin foundation (P1) but is independent +of the OTEL plugin (P2). + +**Independent Test**: Start CGC with the Xdebug plugin enabled in development mode. +Trigger an Xdebug connection from a PHP process. Verify that stack frame nodes appear +in the graph linked to the corresponding static method nodes from a pre-indexed +repository. + +**Acceptance Scenarios**: + +1. **Given** the Xdebug plugin is enabled and a repository is indexed, **When** a PHP + process triggers an Xdebug trace, **Then** the full call stack appears in the graph + as linked frame nodes within 5 seconds of the trace completing. +2. **Given** the same call chain occurs repeatedly, **When** ingestion processes the + repeated traces, **Then** the graph contains deduplicated nodes (no duplicate chains) + and the repetition count is reflected rather than duplicated structure. +3. **Given** a frame resolves to a method that CGC has indexed, **When** the graph is + queried, **Then** the frame node is linked to the corresponding static method node, + enabling navigation from runtime execution to source definition. +4. **Given** the Xdebug plugin is not installed, **When** CGC starts, **Then** no + Xdebug-related commands or tools appear and no port is opened. + +--- + +### User Story 4 - Project Knowledge via Memory Plugin (Priority: P4) + +A developer or AI assistant wants to store and retrieve structured project knowledge +(specifications, decisions, research notes, known bugs) alongside the code graph. When +the memory plugin is enabled, the AI assistant can link stored knowledge entities to +specific classes or methods that CGC has indexed, enabling queries like "show me the +spec for the payment service and which methods implement it" or "which running code has +no associated specification." + +**Why this priority**: The memory plugin uses an existing third-party service with no +custom ingestion logic to build. It delivers high value (project knowledge linked to +code) with the lowest implementation cost of the three data-layer plugins. Its queries +are most powerful in combination with the static layer already provided by core CGC. + +**Independent Test**: Enable the memory plugin. Using the MCP tools it exposes, store a +knowledge entity describing a specific class that exists in an indexed repository. Query +for all classes that have associated knowledge entities and verify the stored entity +appears linked to the correct code node. + +**Acceptance Scenarios**: + +1. **Given** the memory plugin is enabled and a repository is indexed, **When** a user + stores a knowledge entity describing a class, **Then** the entity is linked to the + corresponding graph node and retrievable via an MCP tool query. +2. **Given** knowledge entities exist in the graph, **When** an AI assistant asks "which + code has no associated specification", **Then** the MCP tool returns the set of + indexed code nodes that have no memory entity linked to them. +3. **Given** the memory plugin is not installed, **When** CGC starts, **Then** no + memory-related commands or tools appear and the core graph is unaffected. + +--- + +### User Story 5 - Automated Container Builds via Common CI/CD Pipeline (Priority: P5) + +A maintainer releasing a new version of CGC or any plugin wants every service that +exposes an MCP endpoint to automatically build a versioned, production-ready container +image and publish it to a container registry. The build pipeline is shared across all +services (CGC core, OTEL plugin, Xdebug plugin, memory plugin), so adding a new plugin +service requires minimal CI configuration changes. The resulting images are compatible +with both Docker Compose and Kubernetes deployment patterns. + +**Why this priority**: The CI/CD pipeline enables reliable, reproducible deployment of +the plugin ecosystem. It is independent of the plugin system itself and can be delivered +after the plugins are working locally. It is foundational for anyone wanting to run +CGC-X in a self-hosted or homelab environment. + +**Independent Test**: Trigger the pipeline for a single service (CGC core or the OTEL +plugin). Verify that a tagged container image is built, passes a health-check smoke +test, and is published to the target registry with the correct version tag. Then verify +that the same pipeline configuration can build a second service with only a service +name change. + +**Acceptance Scenarios**: + +1. **Given** a version tag is pushed to the repository, **When** the pipeline runs, + **Then** container images for all enabled plugin services are built and published with + that version tag and a `latest` tag. +2. **Given** a plugin service container is started from its published image, **When** a + health check is performed, **Then** the service responds correctly within 30 seconds. +3. **Given** a new plugin service directory follows the shared conventions, **When** it + is added to the pipeline configuration, **Then** it builds and publishes alongside + existing services without changes to shared pipeline logic. +4. **Given** a build failure occurs in one service, **When** the pipeline runs, **Then** + only that service's build fails; other services complete successfully and their images + are published. +5. **Given** published images, **When** a Kubernetes manifest referencing those images + is applied to a cluster, **Then** the services start successfully and connect to their + configured graph database. + +--- + +### Edge Cases + +- What happens when a plugin depends on a specific graph schema version and the core has + been upgraded with schema changes? +- How does CGC handle a plugin that registers a CLI command name or MCP tool name + already used by another loaded plugin? +- What happens if the graph database is unavailable when a plugin attempts to write + ingested data? +- How does the system behave when the OTEL plugin receives a very high volume of spans + (burst traffic) that exceeds ingestion capacity? +- What happens when Xdebug sends stack frames for a file path that CGC has not indexed? +- How are sensitive values (database credentials, API keys) managed in container images + so they are never baked into the image layer? + +## Requirements *(mandatory)* + +### Functional Requirements + +**Plugin System Core** + +- **FR-001**: CGC MUST provide a plugin registration interface that allows independently + installable packages to declare CLI commands and MCP tools without modifying core code. +- **FR-002**: CGC MUST auto-discover installed plugins at startup and load them without + requiring manual configuration file edits. +- **FR-003**: CGC MUST isolate plugin failures so that a broken or incompatible plugin + does not prevent CGC core or other plugins from starting. +- **FR-004**: CGC MUST enforce plugin version compatibility checks and skip plugins that + declare an unsupported version range, reporting a clear diagnostic message. +- **FR-005**: CGC MUST ensure plugin-registered CLI commands appear in the top-level + help output, grouped under a visible "plugins" section or annotated as plugin-provided. +- **FR-006**: CGC MUST ensure plugin-registered MCP tools appear in the MCP tool listing + alongside core tools with their plugin source identified in metadata. + +**CLI Plugin Interface** + +- **FR-007**: The plugin interface MUST define a standard contract for registering CLI + command groups, including command name, arguments, options, and handler. +- **FR-008**: Plugins MUST be able to add new top-level CLI command groups without + conflicting with core command names. + +**MCP Plugin Interface** + +- **FR-009**: The plugin interface MUST define a standard contract for registering MCP + tools, including tool name, description, input schema, and handler function. +- **FR-010**: Plugins MUST be able to share the same graph database connection managed + by CGC core rather than opening independent connections. + +**OTEL Processor Plugin** + +- **FR-011**: The OTEL plugin MUST expose an ingestion endpoint that accepts telemetry + spans from a standard OpenTelemetry collector. +- **FR-012**: The OTEL plugin MUST extract structured runtime data from spans (service + identity, code namespace, called function, HTTP route, database query) and write it + to the graph as typed runtime nodes and relationships. +- **FR-013**: The OTEL plugin MUST attempt to correlate runtime nodes to existing static + code nodes in the graph where the function identity can be resolved. +- **FR-014**: The OTEL plugin MUST detect and represent cross-service calls as graph + edges between service nodes. +- **FR-015**: The OTEL plugin MUST support configurable span filtering to exclude + high-noise spans (health checks, metrics polling) from graph storage. +- **FR-016**: The OTEL plugin MUST expose at least one MCP tool that enables querying + the execution path for a specific request or route. + +**Xdebug Listener Plugin** + +- **FR-017**: The Xdebug plugin MUST expose a TCP listener that accepts DBGp protocol + connections from Xdebug-enabled PHP processes. +- **FR-018**: The Xdebug plugin MUST capture the full call stack on each trace event + and write stack frame nodes and call-chain relationships to the graph. +- **FR-019**: The Xdebug plugin MUST deduplicate identical call chains so repeated + execution of the same path does not create redundant graph structure. +- **FR-020**: The Xdebug plugin MUST attempt to resolve stack frames to static method + nodes already indexed by CGC core. +- **FR-021**: The Xdebug plugin MUST be configurable as a development/staging-only + service, excluded from production deployments without changing core configuration. + +**Memory Plugin** + +- **FR-022**: The memory plugin MUST expose MCP tools for storing, retrieving, updating, + and searching structured knowledge entities (specifications, decisions, research, + bugs, feature context). +- **FR-023**: The memory plugin MUST allow knowledge entities to be linked to specific + code nodes (classes, methods) already present in the graph. +- **FR-024**: The memory plugin MUST support full-text search across stored knowledge + entities via an MCP query tool. +- **FR-025**: The memory plugin MUST expose an MCP tool that returns all code nodes + lacking any associated knowledge entity, to identify undocumented code. + +**CI/CD Pipeline** + +- **FR-026**: The pipeline MUST build a versioned container image for each plugin + service when a version tag is pushed to the repository. +- **FR-027**: Container images MUST pass a basic health-check smoke test before being + published to the registry. +- **FR-028**: The pipeline MUST publish images with both a specific version tag and a + `latest` tag to the configured container registry. +- **FR-029**: A build failure in one service image MUST NOT prevent other service images + from completing their build and publish steps. +- **FR-030**: The pipeline MUST support a shared build configuration so that adding a + new plugin service requires only adding the service name to a list, not duplicating + pipeline logic. +- **FR-031**: Container images MUST NOT embed sensitive credentials; all secrets MUST + be provided at runtime via environment variables. +- **FR-032**: Each published image MUST include a container health-check definition that + verifies the service is ready to accept connections. +- **FR-033**: Published images MUST be compatible with Kubernetes pod specifications + (no host-mode networking requirements, configurable via environment variables only). + +### Key Entities + +- **Plugin**: A self-contained, independently installable package that contributes CLI + commands and/or MCP tools to CGC. Has a declared name, version, compatibility range, + and lists of registered commands and tools. +- **PluginRegistry**: The runtime component within CGC core that discovers, validates, + and loads installed plugins. Tracks which plugins are active and resolves conflicts. +- **CLICommand**: A command or command group contributed by a plugin. Has a name, + description, argument schema, and an executing handler. +- **MCPTool**: An MCP-protocol tool contributed by a plugin. Has a name, description, + input schema, and a handler. Source plugin is identified in its metadata. +- **RuntimeNode**: A graph node produced by the OTEL or Xdebug plugin representing an + observed execution event (span, stack frame). Carries a `source` property identifying + its origin layer. +- **KnowledgeEntity**: A structured project knowledge record (spec, decision, research, + bug, feature) stored by the memory plugin. Can be linked to static code nodes. +- **ContainerImage**: A versioned, publishable artifact for a plugin service. Produced + by the CI/CD pipeline and tagged with the release version. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: A developer can create a working plugin that adds a CLI command and an MCP + tool to CGC in under 2 hours, using only the published plugin interface documentation + and without reading CGC core source code. +- **SC-002**: Installing or uninstalling a plugin requires no changes to CGC core + configuration files — zero manual edits. +- **SC-003**: CGC with all three plugins enabled starts in under 15 seconds on standard + developer hardware. +- **SC-004**: Runtime span data from an instrumented request appears in the graph within + 10 seconds of the request completing under normal load conditions. +- **SC-005**: An AI assistant using the combined graph (static + runtime + memory) can + answer cross-layer queries (e.g., "what code ran without a spec") that are impossible + with static analysis alone — validated by 5 documented canonical query examples that + all return correct results. +- **SC-006**: The CI/CD pipeline builds and publishes all plugin service images in a + single pipeline run triggered by a version tag — zero manual steps required after + tagging. +- **SC-007**: Any published plugin service image passes its health check within 30 + seconds of container startup. +- **SC-008**: A new plugin service can be added to the CI/CD pipeline by a contributor + who changes only the service list in pipeline configuration — no pipeline logic + changes required. +- **SC-009**: Duplicate call-chain ingestion (the same execution path observed multiple + times) does not increase graph node count — deduplication is 100% effective for + identical chains. +- **SC-010**: All plugin service images run successfully in a Kubernetes environment + using only standard Kubernetes primitives (Deployments, Services, ConfigMaps, Secrets). + +## Assumptions + +- The existing CGC codebase uses Python 3.10+ and the plugin interface will be + implemented in Python using the standard entry-points discovery mechanism. +- The graph database (FalkorDB or Neo4j) is already running and accessible to all + plugins via the connection managed by CGC core. +- Plugin authors are expected to be Python developers familiar with the CGC graph schema. +- The OTEL plugin is the primary runtime layer for production use; Xdebug is dev/staging + only, consistent with the research document's stated intent. +- The memory plugin wraps an existing third-party service (`mcp/neo4j-memory` Docker + image) rather than implementing custom storage logic; the plugin is primarily a + packaging and wiring concern. +- CI/CD pipeline targets GitHub Actions as the execution environment, consistent with + the project's existing workflows. +- Container registry target is determined by project maintainers at implementation time + (Docker Hub, GHCR, or self-hosted). diff --git a/specs/001-cgc-plugin-extension/tasks.md b/specs/001-cgc-plugin-extension/tasks.md new file mode 100644 index 00000000..2bd9876d --- /dev/null +++ b/specs/001-cgc-plugin-extension/tasks.md @@ -0,0 +1,318 @@ +--- + +description: "Task list for CGC Plugin Extension System" +--- + +# Tasks: CGC Plugin Extension System + +**Input**: Design documents from `specs/001-cgc-plugin-extension/` +**Prerequisites**: plan.md ✅ | spec.md ✅ | research.md ✅ | data-model.md ✅ | contracts/ ✅ + +**Tests**: Included — required by Constitution Principle III (Testing Pyramid, NON-NEGOTIABLE). +Tests MUST be written and observed to FAIL before the corresponding implementation task. + +**Organization**: Tasks grouped by user story to enable independent implementation and testing. + +## Format: `[ID] [P?] [Story?] Description` + +- **[P]**: Can run in parallel (different files, no dependencies) +- **[Story]**: Which user story this task belongs to (US1–US5) +- Exact file paths included in every task description + +## Path Conventions + +- Core CGC: `src/codegraphcontext/` +- Plugin packages: `plugins/cgc-plugin-/src/cgc_plugin_/` +- Tests: `tests/unit/plugin/`, `tests/integration/plugin/`, `tests/e2e/plugin/` +- CI/CD: `.github/workflows/`, `.github/services.json` +- Deployment: `docker-compose.yml`, `docker-compose.dev.yml`, `k8s/` + +--- + +## Phase 1: Setup (Shared Infrastructure) + +**Purpose**: Initialize all plugin package scaffolding and root configuration before any +story work begins. + +- [X] T001 Create `plugins/` directory tree: `plugins/cgc-plugin-otel/src/cgc_plugin_otel/`, `plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/`, `plugins/cgc-plugin-memory/src/cgc_plugin_memory/`, `plugins/cgc-plugin-stub/src/cgc_plugin_stub/` with empty `__init__.py` placeholders +- [X] T002 [P] Write `plugins/cgc-plugin-otel/pyproject.toml` — package name `cgc-plugin-otel`, entry-points groups `cgc_cli_plugins` and `cgc_mcp_plugins`, deps: `grpcio>=1.57.0`, `opentelemetry-proto>=0.43b0`, `opentelemetry-sdk>=1.20.0`, `typer[all]>=0.9.0`, `neo4j>=5.15.0` +- [X] T003 [P] Write `plugins/cgc-plugin-xdebug/pyproject.toml` — package name `cgc-plugin-xdebug`, entry-points groups `cgc_cli_plugins` and `cgc_mcp_plugins`, deps: `typer[all]>=0.9.0`, `neo4j>=5.15.0` (stdlib-only implementation) +- [X] T004 [P] Write `plugins/cgc-plugin-memory/pyproject.toml` — package name `cgc-plugin-memory`, entry-points groups `cgc_cli_plugins` and `cgc_mcp_plugins`, deps: `typer[all]>=0.9.0`, `neo4j>=5.15.0` +- [X] T005 [P] Write `plugins/cgc-plugin-stub/pyproject.toml` — package name `cgc-plugin-stub`, entry-points groups `cgc_cli_plugins` and `cgc_mcp_plugins`, dep: `typer[all]>=0.9.0` only (minimal test fixture) +- [X] T006 Add `packaging>=23.0` dependency and optional extras `[otel]`, `[xdebug]`, `[memory]`, `[all]` to root `pyproject.toml`, each extra pointing at its corresponding plugin package in `plugins/` + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +**Purpose**: Core infrastructure that MUST be complete before any user story can be implemented. +The `PluginRegistry` class, graph schema migration, and test infrastructure are shared by all stories. + +**⚠️ CRITICAL**: No user story work can begin until this phase is complete. + +> **NOTE: Write tests FIRST (T008), ensure they FAIL before implementing T007** + +- [X] T007 [P] Add plugin schema constraints and indexes to `config/neo4j/init.cypher` — `UNIQUE` constraints for Service.name, Trace.trace_id, Span.span_id, StackFrame.frame_id; indexes on Span.trace_id, Span.class_name, Span.http_route, StackFrame.fqn; FULLTEXT indexes for Memory.name+entity_type and Observation.content (per data-model.md) +- [X] T008 Write `tests/unit/plugin/test_plugin_registry.py` — unit tests (all entry points mocked) covering: discovers plugins from both entry-point groups, validates PLUGIN_METADATA required fields, skips plugin with incompatible cgc_version_constraint, skips plugin with conflicting name (second plugin), catches ImportError without crashing host, catches exception in get_plugin_commands() without crashing host, reports loaded/failed counts correctly. **Run and confirm FAILING before T009.** +- [X] T009 Implement `src/codegraphcontext/plugin_registry.py` — `PluginRegistry` class with: `discover_cli_plugins()` (reads `cgc_cli_plugins` group), `discover_mcp_plugins()` (reads `cgc_mcp_plugins` group), `_validate_metadata()` (checks required fields + cgc_version_constraint via `packaging.specifiers.SpecifierSet`), `_safe_load()` (try/except + SIGALRM 5s timeout on Unix), `_safe_call()` (try/except wrapper for get_plugin_commands/get_mcp_tools/get_mcp_handlers), `loaded_plugins: dict`, `failed_plugins: dict`, startup summary log line +- [X] T010 Update `tests/run_tests.sh` to include `tests/unit/plugin/` and `tests/integration/plugin/` in the `fast` suite alongside existing unit + integration paths + +**Checkpoint**: PluginRegistry unit tests pass. Schema migration ready. Fast suite covers plugin tests. + +--- + +## Phase 3: User Story 1 — Plugin Extensibility Foundation (Priority: P1) 🎯 MVP + +**Goal**: CGC discovers and loads installed plugins automatically; CLI and MCP both surface +plugin-contributed commands and tools; broken plugins never crash the host process. + +**Independent Test**: `pip install -e plugins/cgc-plugin-stub` → `cgc --help` shows `stub` +command group → `cgc stub hello` works → MCP tool `stub_hello` appears in tools/list → +`pip uninstall cgc-plugin-stub` → CGC restarts cleanly with no stub artifacts. + +> **NOTE: Write integration tests (T011) FIRST, ensure they FAIL before T012–T015** + +- [X] T011 Write `tests/integration/plugin/test_plugin_load.py` — integration tests using the stub plugin (installed as editable in conftest fixture): stub CLI command appears in `app.registered_commands` after registry runs; stub MCP tool name appears in server.tools dict; second incompatible-version stub is skipped with warning; two conflicting-name stubs load only first; registry reports correct counts. **Run and confirm FAILING before T012.** +- [X] T012 [P] [US1] Implement `plugins/cgc-plugin-stub/src/cgc_plugin_stub/__init__.py` — `PLUGIN_METADATA` dict: name `cgc-plugin-stub`, version `0.1.0`, cgc_version_constraint `>=0.1.0`, description `Stub plugin for testing` +- [X] T013 [P] [US1] Implement `plugins/cgc-plugin-stub/src/cgc_plugin_stub/cli.py` — `get_plugin_commands()` returning `("stub", stub_app)` where `stub_app` has one command `hello` that echoes "Hello from stub plugin" +- [X] T014 [P] [US1] Implement `plugins/cgc-plugin-stub/src/cgc_plugin_stub/mcp_tools.py` — `get_mcp_tools()` returning one tool `stub_hello` with inputSchema `{name: string}`; `get_mcp_handlers()` returning handler that returns `{"greeting": f"Hello {name}"}` +- [X] T015 [US1] Modify `src/codegraphcontext/cli/main.py` — add `_load_plugin_cli_commands(registry: PluginRegistry)` function that calls `app.add_typer()` for each entry in `registry.loaded_plugins`; call at module startup after core command registration; add `cgc plugin list` sub-command showing loaded/failed plugins with name, version, tool count +- [X] T016 [US1] Modify `src/codegraphcontext/server.py` — instantiate `PluginRegistry` in `MCPServer.__init__()`, call `_load_plugin_tools()` that merges plugin tool definitions into `self.tools` dict (with conflict check), store plugin handlers in `self.plugin_tool_handlers: dict`, update `handle_tool_call()` to check `self.plugin_tool_handlers` before built-in handler map + +**Checkpoint**: `pip install -e plugins/cgc-plugin-stub && cgc plugin list` shows stub; MCP tools/list includes `stub_hello`; uninstall leaves CGC clean. + +--- + +## Phase 4: User Story 2 — Runtime Intelligence via OTEL Plugin (Priority: P2) + +**Goal**: OTEL plugin receives telemetry spans, writes Service/Trace/Span nodes to the +graph, correlates spans to static Method nodes, and exposes MCP tools for runtime queries. + +**Independent Test**: With a pre-indexed PHP repository, send a synthetic OTLP span payload +to the OTEL plugin endpoint. Query `MATCH (s:Span) RETURN count(s)` → non-zero. +Query `MATCH (s:Span)-[:CORRELATES_TO]->(m:Method) RETURN s.name, m.fqn LIMIT 5` → returns linked results. + +> **NOTE: Write unit tests (T017) FIRST, ensure they FAIL before T018–T022** + +- [X] T017 Write `tests/unit/plugin/test_otel_processor.py` — unit tests (mocked db_manager, no gRPC): `extract_php_context()` parses code.namespace+code.function into fqn; `extract_php_context()` handles missing attributes gracefully (returns None fqn); `is_cross_service_span()` returns True for CLIENT kind spans with peer.service set; `should_filter_span()` returns True for health-check routes matching config; `build_span_dict()` computes duration_ms correctly from ns timestamps. **Run and confirm FAILING before T018.** +- [X] T018 [P] [US2] Implement `plugins/cgc-plugin-otel/src/cgc_plugin_otel/__init__.py` — `PLUGIN_METADATA` dict: name `cgc-plugin-otel`, version `0.1.0`, cgc_version_constraint `>=0.1.0` +- [X] T019 [P] [US2] Implement `plugins/cgc-plugin-otel/src/cgc_plugin_otel/cli.py` — `get_plugin_commands()` returning `("otel", otel_app)` with commands: `query-spans --route TEXT --limit INT`, `list-services`, `status` (shows whether receiver is running) +- [X] T020 [US2] Implement `plugins/cgc-plugin-otel/src/cgc_plugin_otel/span_processor.py` — `extract_php_context(span_attrs: dict) -> dict` (parses code.namespace, code.function, http.route, http.method, db.statement, db.system into typed dict); `build_fqn(namespace, function) -> str | None`; `is_cross_service_span(span_kind, span_attrs) -> bool`; `should_filter_span(span_attrs, filter_routes: list[str]) -> bool` (configurable noise filter) +- [X] T021 [US2] Implement `plugins/cgc-plugin-otel/src/cgc_plugin_otel/neo4j_writer.py` — `AsyncOtelWriter` class: async `write_batch(spans: list[dict])` using `asyncio.Queue(maxsize=10000)` and periodic flush (batch size 100, timeout 5s); MERGE queries for Service, Trace, Span nodes; CHILD_OF (parent_span_id), PART_OF (trace), ORIGINATED_FROM (service), CALLS_SERVICE (CLIENT kind), CORRELATES_TO (fqn match against existing Method nodes); dead-letter queue with `asyncio.Queue(maxsize=100000)` for Neo4j unavailability; `_background_retry_task()` coroutine +- [X] T022 [US2] Implement `plugins/cgc-plugin-otel/src/cgc_plugin_otel/receiver.py` — `OTLPSpanReceiver` class implementing `TraceServiceServicer` (grpcio + opentelemetry-proto); `Export()` method queues spans for batch processing; `main()` starts gRPC server on `OTEL_RECEIVER_PORT` (default 5317) + launches `process_span_batch()` background task; graceful shutdown on SIGTERM +- [X] T023 [P] [US2] Implement `plugins/cgc-plugin-otel/src/cgc_plugin_otel/mcp_tools.py` — `get_mcp_tools()` returning: `otel_query_spans` (args: http_route, service, limit), `otel_list_services` (no args), `otel_cross_layer_query` (args: query_type enum: `unspecced_running_code|cross_service_calls|recent_executions`); `get_mcp_handlers()` with corresponding Cypher-backed handlers using `server_context["db_manager"]` +- [X] T024 [US2] Create `config/otel-collector/config.yaml` — OTLP gRPC+HTTP receivers (ports 4317, 4318); batch processor (timeout 5s, send_batch_size 512); filter processor dropping spans where `http.route` matches `/health`, `/metrics`, `/ping`; OTLP exporter forwarding to `otel-processor:5317` (insecure TLS) +- [X] T025 [US2] Add OTEL services to `docker-compose.yml` — `otel-collector` service (image: `otel/opentelemetry-collector-contrib:latest`, ports 4317-4318, depends on otel-processor); `cgc-otel-processor` service (build: `plugins/cgc-plugin-otel`, env: NEO4J_URI/USERNAME/PASSWORD/LISTEN_PORT/LOG_LEVEL, depends on neo4j healthcheck, Traefik labels) +- [X] T026 [US2] Write `tests/integration/plugin/test_otel_integration.py` — with real Neo4j fixture (or mock db_manager): call `write_batch()` with synthetic span dicts; assert Service node created with correct name; assert Span node created with correct span_id; assert CHILD_OF relationship created for parent_span_id; assert CORRELATES_TO created when fqn matches pre-existing Method node; assert filtered spans (health route) produce zero graph nodes + +**Checkpoint**: OTEL plugin loads, gRPC receiver accepts a synthetic span, Service+Span nodes appear in graph with CORRELATES_TO link to static Method. + +--- + +## Phase 5: User Story 3 — Development Traces via Xdebug Plugin (Priority: P3) + +**Goal**: Xdebug plugin runs a TCP DBGp listener, captures PHP call stacks, deduplicates +chains, writes StackFrame nodes to the graph, and links frames to static Method nodes. + +**Independent Test**: With a pre-indexed PHP repository, simulate a DBGp TCP connection +sending a synthetic stack_get XML response. Verify StackFrame nodes appear in the graph +with CALLED_BY chain relationships and RESOLVES_TO links to Method nodes. + +> **NOTE: Write unit tests (T027) FIRST, ensure they FAIL before T028–T031** + +- [X] T027 Write `tests/unit/plugin/test_xdebug_parser.py` — unit tests (no TCP): `parse_stack_xml(xml_str) -> list[dict]` returns correct frame list from sample DBGp XML; `compute_chain_hash(frames) -> str` returns same hash for identical frame lists and different hash for different lists; `build_frame_id(class_name, method_name, file_path, line) -> str` returns deterministic unique string; dedup check returns True for hash in LRU cache and False for new hash. **Run and confirm FAILING before T028.** +- [X] T028 [P] [US3] Implement `plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/__init__.py` — `PLUGIN_METADATA` dict: name `cgc-plugin-xdebug`, version `0.1.0`, cgc_version_constraint `>=0.1.0`; note in description that this is dev/staging only +- [X] T029 [P] [US3] Implement `plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/cli.py` — `get_plugin_commands()` returning `("xdebug", xdebug_app)` with commands: `start` (starts listener, requires `CGC_PLUGIN_XDEBUG_ENABLED=true`), `stop`, `status`, `list-chains --limit INT` +- [X] T030 [US3] Implement `plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/dbgp_server.py` — `DBGpServer` class: `listen(host, port)` opens TCP socket with `SO_REUSEADDR`; `handle_connection(conn)` reads DBGp init packet, sends `run` command, loops: sends `stack_get -i {seq}`, parses XML response via `parse_stack_xml()`, calls `neo4j_writer.write_chain()`, sends `run`; `parse_stack_xml(xml: str) -> list[dict]` using `xml.etree.ElementTree`; server only starts when env var `CGC_PLUGIN_XDEBUG_ENABLED=true` +- [X] T031 [US3] Implement `plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/neo4j_writer.py` — `XdebugWriter` class: `lru_cache: dict[str, int]` (hash → observation_count, max `DEDUP_CACHE_SIZE=10000`); `write_chain(frames: list[dict], db_manager)`: computes chain_hash, checks LRU — if seen, increments observation_count on existing StackFrame and returns; else MERGEs StackFrame nodes for each frame, creates CALLED_BY chain from depth ordering, attempts RESOLVES_TO match against `Method {fqn: $fqn}` for each frame +- [X] T032 [P] [US3] Implement `plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/mcp_tools.py` — `get_mcp_tools()` returning: `xdebug_list_chains` (args: limit, min_observations), `xdebug_query_chain` (args: class_name, method_name); `get_mcp_handlers()` with Cypher-backed handlers +- [X] T033 [US3] Add `xdebug-listener` service to `docker-compose.dev.yml` — build: `plugins/cgc-plugin-xdebug`, env: NEO4J_URI/USERNAME/PASSWORD/LISTEN_HOST/LISTEN_PORT=9003/DEDUP_CACHE_SIZE/LOG_LEVEL=DEBUG/CGC_PLUGIN_XDEBUG_ENABLED=true, ports: `9003:9003`, depends on neo4j healthcheck + +**Checkpoint**: Xdebug plugin loads with `CGC_PLUGIN_XDEBUG_ENABLED=true`, synthetic DBGp XML input produces StackFrame nodes with CALLED_BY chain and RESOLVES_TO Method links. + +--- + +## Phase 6: User Story 4 — Project Knowledge via Memory Plugin (Priority: P4) + +**Goal**: Memory plugin exposes MCP tools and CLI commands to store/search/link knowledge +entities in the same Neo4j graph, enabling "which code has no spec?" queries. + +**Independent Test**: `cgc memory store --type spec --name "Order spec" --content "..." +--links-to "App\\Http\\Controllers\\OrderController"` → Memory node in graph → +`cgc memory undocumented` returns unlinked Class nodes. + +> **NOTE: Write integration tests (T034) FIRST, ensure they FAIL before T035–T037** + +- [X] T034 Write `tests/integration/plugin/test_memory_integration.py` — tests with mocked db_manager running real Cypher: call `memory_store` handler → assert Memory node created with correct entity_type; call `memory_link` with existing Class fqn → assert DESCRIBES relationship created; call `memory_undocumented` → assert Class nodes without DESCRIBES appear in result; call `memory_search` with text → assert full-text search returns matching Memory node. **Run and confirm FAILING before T035.** +- [X] T035 [P] [US4] Implement `plugins/cgc-plugin-memory/src/cgc_plugin_memory/__init__.py` — `PLUGIN_METADATA` dict: name `cgc-plugin-memory`, version `0.1.0`, cgc_version_constraint `>=0.1.0` +- [X] T036 [P] [US4] Implement `plugins/cgc-plugin-memory/src/cgc_plugin_memory/cli.py` — `get_plugin_commands()` returning `("memory", memory_app)` with commands: `store --type TEXT --name TEXT --content TEXT [--links-to TEXT]`, `search --query TEXT`, `undocumented [--type TEXT]`, `status` +- [X] T037 [US4] Implement `plugins/cgc-plugin-memory/src/cgc_plugin_memory/mcp_tools.py` — `get_mcp_tools()` returning: `memory_store` (args: entity_type, name, content, links_to?), `memory_search` (args: query, limit), `memory_undocumented` (args: node_type enum Class|Method, limit), `memory_link` (args: memory_id, node_fqn, node_type); `get_mcp_handlers()` with Cypher-backed handlers: memory_store MERGEs Memory node + HAS_OBSERVATION + optional DESCRIBES; memory_search uses FULLTEXT index; memory_undocumented matches Class/Method WHERE NOT EXISTS DESCRIBES; memory_link creates DESCRIBES edge +- [X] T038 [US4] Add `cgc-memory` service to `docker-compose.yml` — image: `mcp/neo4j-memory`, env: NEO4J_URL/NEO4J_USERNAME/NEO4J_PASSWORD/NEO4J_DATABASE=neo4j/NEO4J_MCP_SERVER_HOST/NEO4J_MCP_SERVER_PORT=8766, depends on neo4j healthcheck, Traefik labels for `memory.${DOMAIN}` + +**Checkpoint**: Memory plugin loads; `memory_store` MCP tool creates Memory+Observation nodes in graph; `memory_undocumented` returns correct unlinked code nodes. + +--- + +## Phase 7: User Story 5 — Automated Container Builds via Common CI/CD Pipeline (Priority: P5) + +**Goal**: GitHub Actions matrix pipeline builds, smoke tests, and publishes versioned Docker +images for all plugin services. Adding a new service requires only editing `.github/services.json`. + +**Independent Test**: Push a test tag; verify GitHub Actions builds all services in parallel; +verify each image's smoke test passes; verify images are tagged with semver + `latest`; +verify a failure in one service does not cancel other builds. + +- [X] T039 [P] [US5] Create `plugins/cgc-plugin-otel/Dockerfile` — `FROM python:3.12-slim`, non-root `USER cgc`, `COPY` and `pip install --no-cache-dir`, `EXPOSE 5317`, `HEALTHCHECK --interval=30s --timeout=10s CMD python -c "import grpc; print('ok')"`, `CMD ["python", "-m", "cgc_plugin_otel.receiver"]`; no `ENV` with secret values +- [X] T040 [P] [US5] Create `plugins/cgc-plugin-xdebug/Dockerfile` — `FROM python:3.12-slim`, non-root user, `EXPOSE 9003`, `HEALTHCHECK CMD python -c "import socket; socket.socket()"`, `CMD ["python", "-m", "cgc_plugin_xdebug.dbgp_server"]`; requires `CGC_PLUGIN_XDEBUG_ENABLED=true` at runtime +- [X] T041 [P] [US5] Create `plugins/cgc-plugin-memory/Dockerfile` — `FROM python:3.12-slim`, non-root user, install `cgc-plugin-memory` package, `EXPOSE 8766`, `HEALTHCHECK --interval=30s CMD python -c "import cgc_plugin_memory; print('ok')"`, env-var-only config +- [X] T042 [US5] Create `.github/services.json` — JSON array with entries for: `cgc-core` (path: `.`, dockerfile: `Dockerfile`, health_check: `version`), `cgc-plugin-otel` (path: `plugins/cgc-plugin-otel`, health_check: `grpc_ping`), `cgc-plugin-memory` (path: `plugins/cgc-plugin-memory`, health_check: `http_health`) per `contracts/cicd-pipeline.md` schema +- [X] T043 [US5] Create `.github/workflows/docker-publish.yml` — `setup` job reads `.github/services.json` and outputs matrix; `build-images` job with `strategy: {matrix: ${{ fromJson(...) }}, fail-fast: false}`: checkout, `docker/setup-buildx-action@v3`, `docker/login-action@v3` (GHCR, skipped on PR), `docker/metadata-action@v5` (semver+latest tags), `docker/build-push-action@v5` with `push: false` + `outputs: type=docker` for smoke test, smoke test per `health_check` type, then `docker/build-push-action@v5` with `push: true` if not PR and smoke test passed; `build-summary` job reports overall status +- [X] T044 [P] [US5] Create `.github/workflows/test-plugins.yml` — GitHub Actions workflow triggered on PR: matrix over plugin directories, runs `pip install -e . -e plugins/${{ matrix.plugin }}` then `pytest tests/unit/plugin/ tests/integration/plugin/ -v` per plugin; fail-fast: false +- [X] T045 [P] [US5] Create `k8s/cgc-plugin-otel/deployment.yaml` — standard `Deployment` (replicas: 1, image ref from registry, env from ConfigMap `cgc-config` for NEO4J_URI/USERNAME + Secret `cgc-secrets` for NEO4J_PASSWORD, readinessProbe via exec checking grpc import, no hostNetwork) +- [X] T046 [P] [US5] Create `k8s/cgc-plugin-otel/service.yaml` — `ClusterIP` Service exposing port 5317 (gRPC receiver) and 4318 (HTTP, forwarded from collector) +- [X] T047 [P] [US5] Create `k8s/cgc-plugin-memory/deployment.yaml` and `k8s/cgc-plugin-memory/service.yaml` — Deployment with `mcp/neo4j-memory` image, env from ConfigMap+Secret, Service exposing port 8766 + +**Checkpoint**: Triggering the workflow on a test tag builds all services in parallel; one intentional Dockerfile error only fails that service's job; remaining images publish to registry with correct semver tags. + +--- + +## Final Phase: Polish & Cross-Cutting Concerns + +**Purpose**: E2E validation, cross-layer queries documentation, and developer experience +improvements that span multiple user stories. + +- [X] T048 Write `tests/e2e/plugin/test_plugin_lifecycle.py` — full user journey E2E test: install stub plugin editable → cgc starts with stub command → cgc plugin list shows stub → stub MCP tool appears in tools/list → call stub_hello via MCP → uninstall stub → cgc restarts cleanly; also: install otel plugin → start receiver → call write_batch with synthetic spans → cross-layer Cypher query returns results; run with `./tests/run_tests.sh e2e` +- [X] T049 [P] Create `docs/plugins/cross-layer-queries.md` — 5 canonical cross-layer Cypher queries validating SC-005: (1) execution path for route, (2) recent methods with no spec, (3) cross-service call chains, (4) specs describing recently-active code, (5) static code never observed at runtime; include expected result schema for each +- [X] T050 [P] Create `docs/plugins/authoring-guide.md` — minimal plugin authoring guide referencing `contracts/plugin-interface.md` and `plugins/cgc-plugin-stub/` as the worked example; covers: package scaffold, PLUGIN_METADATA, CLI contract, MCP contract, testing, publishing to PyPI +- [X] T051 [P] Update root `CLAUDE.md` agent context with new plugin directories, plugin entry-point groups (`cgc_cli_plugins`, `cgc_mcp_plugins`), and the `plugins/` layout — run `.specify/scripts/bash/update-agent-context.sh claude` +- [X] T052 Run full `quickstart.md` validation: install all three plugins editable, execute every command in `specs/001-cgc-plugin-extension/quickstart.md` end-to-end, verify all succeed; update quickstart if any step is incorrect + +--- + +## Dependencies & Execution Order + +### Phase Dependencies + +- **Setup (Phase 1)**: No dependencies — start immediately; T002-T005 in parallel +- **Foundational (Phase 2)**: Depends on Setup (T001 for dirs, T006 for root pyproject) + - T007 (schema) and T010 (test runner) are independent of each other — run in parallel + - T008 (unit tests) must be written before T009 (PluginRegistry implementation) +- **US1 (Phase 3)**: Depends on Foundational (T009 PluginRegistry complete) + - T011 (integration tests) written before T012-T016 + - T012, T013, T014 (stub plugin files) independent of each other — run in parallel + - T015 and T016 (core modifications) can run in parallel once T009 is done +- **US2 (Phase 4)**: Depends on US1 complete (plugin loading infrastructure) + - T017 (unit tests) before T018-T023 + - T018, T019 (metadata + CLI) independent — parallel + - T020 → T021 → T022 (processor → writer → receiver — sequential) + - T023 (MCP tools) independent of T020-T022 — can run in parallel with T021 + - T024 (OTel Collector config), T025 (docker-compose) independent — parallel after T022 +- **US3 (Phase 5)**: Depends on US1 complete; independent of US2 + - T027 (unit tests) before T028-T032 + - T028, T029 (metadata + CLI) parallel + - T030 → T031 (dbgp_server → neo4j_writer — sequential; writer depends on parsed frames) + - T032 (MCP tools) independent — parallel with T031 +- **US4 (Phase 6)**: Depends on US1 complete; independent of US2 and US3 + - T034 (integration tests) before T035-T037 + - T035, T036 (metadata + CLI) parallel + - T037 (MCP tools) depends on T035 (metadata) +- **US5 (Phase 7)**: Depends on US2 and US3 complete (Dockerfiles need working services) + - T039, T040, T041 (Dockerfiles) all parallel + - T042 (services.json) before T043 (workflow) + - T044 (test workflow) parallel with T043 + - T045, T046, T047 (K8s manifests) all parallel, independent of T043-T044 +- **Polish (Final Phase)**: Depends on all user stories complete + - T049, T050, T051 all parallel + - T052 (quickstart validation) last — sequentially after T048-T051 + +### User Story Dependencies + +- **US1 (P1)**: No story dependencies — first to implement +- **US2 (P2)**: Depends on US1 complete +- **US3 (P3)**: Depends on US1 complete — independent of US2 +- **US4 (P4)**: Depends on US1 complete — independent of US2, US3 +- **US5 (P5)**: Depends on US2 + US3 complete (container services need working implementations) + +### Within Each User Story + +- Unit/integration tests MUST be written and FAIL before corresponding implementation +- `__init__.py` (metadata) before CLI and MCP modules +- CLI and MCP modules can be written in parallel +- Core logic (processor, writer, server) before MCP handlers that use it +- Docker/compose additions after core implementation is working + +--- + +## Parallel Execution Examples + +### Phase 1 (Setup) +``` +Parallel: T002, T003, T004, T005 — four plugin pyproject.toml files, different paths +Then: T001 (dirs), T006 (root pyproject) +``` + +### Phase 2 (Foundational) +``` +Parallel: T007 (schema migration), T010 (test runner update) +Sequential: T008 (write unit tests) → T009 (implement PluginRegistry) +``` + +### US2 (OTEL Plugin) +``` +Write + fail: T017 +Parallel: T018, T019 (metadata + CLI) +Sequential: T020 → T021 → T022 (processor → writer → receiver) +Parallel with T021: T023 (MCP tools — uses db_manager directly, not receiver) +Parallel: T024, T025 (config + docker-compose) +Then: T026 (integration tests) +``` + +### US5 (CI/CD) +``` +Parallel: T039, T040, T041 (three Dockerfiles) +Sequential: T042 → T043 (services.json must exist before workflow reads it) +Parallel: T044, T045, T046, T047 (test workflow + K8s manifests) +``` + +--- + +## Implementation Strategy + +### MVP First (US1 Only) + +1. Complete Phase 1: Setup (T001–T006) +2. Complete Phase 2: Foundational (T007–T010) +3. Complete Phase 3: US1 Plugin Foundation (T011–T016) +4. **STOP and VALIDATE**: `cgc plugin list` works; stub plugin loads; MCP tools list includes stub; broken plugin doesn't crash CGC +5. Deploy/demo: plugin system is usable by third-party authors + +### Incremental Delivery + +1. Setup + Foundational → PluginRegistry ready +2. US1 → Plugin system works → **demo: install any plugin** +3. US2 → Runtime intelligence → **demo: "show what ran during this request"** +4. US3 → Dev traces → **demo: "show concrete implementations that ran"** +5. US4 → Project knowledge → **demo: "which code has no spec?"** +6. US5 → CI/CD → **demo: `git tag v0.1.0` builds all images automatically** + +### Parallel Team Strategy + +With 3 developers after US1 is complete: +- Developer A: US2 (OTEL Plugin) +- Developer B: US3 (Xdebug Plugin) +- Developer C: US4 (Memory Plugin) + +All three complete independently, then US5 (CI/CD) begins. + +--- + +## Notes + +- `[P]` tasks = different files, no dependencies on incomplete tasks in the same phase +- `[US?]` maps each task to its user story for traceability and independent delivery +- Tests MUST be written and FAIL before implementation — this is NON-NEGOTIABLE per Constitution Principle III +- Each phase has a named Checkpoint — validate before moving to the next phase +- Verify `./tests/run_tests.sh fast` passes after completing each phase +- Plugin name prefix convention for MCP tools: `_` (e.g., `otel_query_spans`) +- No credentials in Dockerfiles or docker-compose.yml — all via environment variables +- Xdebug plugin: requires `CGC_PLUGIN_XDEBUG_ENABLED=true` at runtime; absent = no TCP port opened diff --git a/src/codegraphcontext/cli/main.py b/src/codegraphcontext/cli/main.py index 9374662e..3b4b4840 100644 --- a/src/codegraphcontext/cli/main.py +++ b/src/codegraphcontext/cli/main.py @@ -24,6 +24,7 @@ from codegraphcontext.server import MCPServer from codegraphcontext.core.database import DatabaseManager +from codegraphcontext.plugin_registry import PluginRegistry from .setup_wizard import run_neo4j_setup_wizard, configure_mcp_client from . import config_manager # Import the new helper functions @@ -102,6 +103,58 @@ def get_version() -> str: mcp_app = typer.Typer(help="MCP client configuration commands") app.add_typer(mcp_app, name="mcp") +# --------------------------------------------------------------------------- +# Plugin CLI integration +# --------------------------------------------------------------------------- + +_plugin_registry: PluginRegistry | None = None + +plugin_app = typer.Typer(help="Manage CGC plugins.") +app.add_typer(plugin_app, name="plugin") + + +@plugin_app.command("list") +def plugin_list(): + """Show all loaded and failed plugins.""" + global _plugin_registry + if _plugin_registry is None: + console.print("[yellow]Plugin registry not initialised.[/yellow]") + return + + table = Table(title="CGC Plugins", box=box.SIMPLE) + table.add_column("Name", style="cyan") + table.add_column("Status", style="bold") + table.add_column("Version") + table.add_column("Tools / Command") + table.add_column("Reason", style="dim") + + for name, info in _plugin_registry.loaded_plugins.items(): + meta = info.get("metadata", {}) + version = meta.get("version", "?") + tools = ", ".join(info.get("mcp_tools", [])) + cmd = info.get("cli_command", "") + detail = tools or cmd or "—" + table.add_row(name, "[green]loaded[/green]", version, detail, "") + + for name, reason in _plugin_registry.failed_plugins.items(): + table.add_row(name, "[red]failed[/red]", "—", "—", reason) + + console.print(table) + + +def _load_plugin_cli_commands(registry: PluginRegistry) -> None: + """Attach plugin-contributed Typer command groups to the root app.""" + global _plugin_registry + _plugin_registry = registry + for cmd_name, typer_app in registry.cli_commands: + app.add_typer(typer_app, name=cmd_name) + + +# Discover and register plugin CLI commands at import time. +_registry = PluginRegistry() +_registry.discover_cli_plugins() +_load_plugin_cli_commands(_registry) + @mcp_app.command("setup") def mcp_setup(): """ diff --git a/src/codegraphcontext/plugin_registry.py b/src/codegraphcontext/plugin_registry.py new file mode 100644 index 00000000..999d3e94 --- /dev/null +++ b/src/codegraphcontext/plugin_registry.py @@ -0,0 +1,283 @@ +""" +Plugin registry for CodeGraphContext. + +Discovers and loads plugins declared via Python entry points: + - Group ``cgc_cli_plugins``: plugins contributing Typer CLI command groups + - Group ``cgc_mcp_plugins``: plugins contributing MCP tools + +Plugins are isolated: a broken plugin logs a warning and is skipped without +affecting CGC core or other plugins. +""" +from __future__ import annotations + +import logging +import signal +import sys +from typing import Any + +from importlib.metadata import entry_points, version as pkg_version, PackageNotFoundError +from packaging.specifiers import SpecifierSet, InvalidSpecifier + +logger = logging.getLogger(__name__) + +_REQUIRED_METADATA_FIELDS = ("name", "version", "cgc_version_constraint", "description") +_LOAD_TIMEOUT_SECONDS = 5 + + +def _get_cgc_version() -> str: + try: + return pkg_version("codegraphcontext") + except PackageNotFoundError: + return "0.0.0" + + +class PluginRegistry: + """ + Discovers, validates, and loads CGC plugins at startup. + + Usage:: + + registry = PluginRegistry() + registry.discover_cli_plugins() # populates cli_commands + registry.discover_mcp_plugins(ctx) # populates mcp_tools + mcp_handlers + + Results are available via: + - ``registry.cli_commands`` list of (name, typer.Typer) + - ``registry.mcp_tools`` dict of tool_name → ToolDefinition + - ``registry.mcp_handlers`` dict of tool_name → callable + - ``registry.loaded_plugins`` dict of name → registration info + - ``registry.failed_plugins`` dict of name → failure reason + """ + + def __init__(self) -> None: + self.cli_commands: list[tuple[str, Any]] = [] + self.mcp_tools: dict[str, dict] = {} + self.mcp_handlers: dict[str, Any] = {} + self.loaded_plugins: dict[str, dict] = {} + self.failed_plugins: dict[str, str] = {} + self._cgc_version = _get_cgc_version() + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + def discover_cli_plugins(self) -> None: + """Discover and load all ``cgc_cli_plugins`` entry points.""" + eps = self._get_entry_points("cgc_cli_plugins") + for ep in eps: + self._load_cli_plugin(ep) + self._log_summary() + + def discover_mcp_plugins(self, server_context: dict | None = None) -> None: + """Discover and load all ``cgc_mcp_plugins`` entry points.""" + if server_context is None: + server_context = {} + eps = self._get_entry_points("cgc_mcp_plugins") + for ep in eps: + self._load_mcp_plugin(ep, server_context) + + # ------------------------------------------------------------------ + # Internal loaders + # ------------------------------------------------------------------ + + def _load_cli_plugin(self, ep: Any) -> None: + plugin_name = ep.name + mod = self._safe_import(plugin_name, ep) + if mod is None: + return + + # Validate metadata + reason = self._validate_metadata(plugin_name, mod) + if reason: + self.failed_plugins[plugin_name] = reason + logger.warning("Plugin '%s' skipped: %s", plugin_name, reason) + return + + # Check for name conflict + if plugin_name in self.loaded_plugins: + msg = f"name conflict with already-loaded plugin '{plugin_name}'" + self.failed_plugins[plugin_name + "_duplicate"] = msg + logger.warning("Plugin '%s' (second instance) skipped: %s", plugin_name, msg) + return + + # Call get_plugin_commands() + get_cmds = getattr(mod, "get_plugin_commands", None) + if get_cmds is None: + reason = "missing get_plugin_commands() function" + self.failed_plugins[plugin_name] = reason + logger.warning("Plugin '%s' skipped: %s", plugin_name, reason) + return + + result = self._safe_call(plugin_name, get_cmds) + if result is None: + return + + try: + cmd_name, typer_app = result + except (TypeError, ValueError) as exc: + reason = f"get_plugin_commands() returned invalid format: {exc}" + self.failed_plugins[plugin_name] = reason + logger.warning("Plugin '%s' skipped: %s", plugin_name, reason) + return + + self.cli_commands.append((cmd_name, typer_app)) + self.loaded_plugins[plugin_name] = { + "status": "loaded", + "metadata": mod.PLUGIN_METADATA, + "cli_command": cmd_name, + } + logger.info("Plugin '%s' loaded CLI command group '%s'", plugin_name, cmd_name) + + def _load_mcp_plugin(self, ep: Any, server_context: dict) -> None: + plugin_name = ep.name + mod = self._safe_import(plugin_name, ep) + if mod is None: + return + + reason = self._validate_metadata(plugin_name, mod) + if reason: + self.failed_plugins[plugin_name] = reason + logger.warning("Plugin '%s' skipped: %s", plugin_name, reason) + return + + get_tools = getattr(mod, "get_mcp_tools", None) + get_handlers = getattr(mod, "get_mcp_handlers", None) + + if get_tools is None: + reason = "missing get_mcp_tools() function" + self.failed_plugins[plugin_name] = reason + logger.warning("Plugin '%s' skipped: %s", plugin_name, reason) + return + + tools = self._safe_call(plugin_name, get_tools, server_context) + if tools is None: + return + + handlers: dict = {} + if get_handlers is not None: + h = self._safe_call(plugin_name, get_handlers, server_context) + if h is not None: + handlers = h + + registered = 0 + for tool_name, tool_def in tools.items(): + if tool_name in self.mcp_tools: + logger.warning( + "Plugin '%s': tool '%s' conflicts with existing tool — skipped", + plugin_name, tool_name, + ) + continue + self.mcp_tools[tool_name] = tool_def + if tool_name in handlers: + self.mcp_handlers[tool_name] = handlers[tool_name] + registered += 1 + + if plugin_name not in self.loaded_plugins: + self.loaded_plugins[plugin_name] = { + "status": "loaded", + "metadata": mod.PLUGIN_METADATA, + "mcp_tools": list(tools.keys()), + } + else: + self.loaded_plugins[plugin_name]["mcp_tools"] = list(tools.keys()) + + logger.info("Plugin '%s' loaded %d MCP tool(s)", plugin_name, registered) + + # ------------------------------------------------------------------ + # Helpers + # ------------------------------------------------------------------ + + def _get_entry_points(self, group: str) -> list: + try: + return list(entry_points(group=group)) + except Exception as exc: + logger.error("Failed to query entry points for group '%s': %s", group, exc) + return [] + + def _safe_import(self, plugin_name: str, ep: Any) -> Any | None: + """Load an entry point with timeout and full exception isolation.""" + _alarm_set = False + try: + if hasattr(signal, "SIGALRM"): + def _timeout_handler(signum, frame): + raise TimeoutError( + f"Plugin '{plugin_name}' import timed out after " + f"{_LOAD_TIMEOUT_SECONDS}s" + ) + signal.signal(signal.SIGALRM, _timeout_handler) + signal.alarm(_LOAD_TIMEOUT_SECONDS) + _alarm_set = True + + mod = ep.load() + return mod + + except TimeoutError as exc: + reason = str(exc) + self.failed_plugins[plugin_name] = reason + logger.error("Plugin '%s' load timeout: %s", plugin_name, reason) + return None + except ImportError as exc: + reason = f"ImportError: {exc}" + self.failed_plugins[plugin_name] = reason + logger.error("Plugin '%s' import failed (missing dependency?): %s", plugin_name, exc) + return None + except AttributeError as exc: + reason = f"AttributeError: {exc}" + self.failed_plugins[plugin_name] = reason + logger.error("Plugin '%s' entry point invalid (bad module path?): %s", plugin_name, exc) + return None + except Exception as exc: + reason = f"{type(exc).__name__}: {exc}" + self.failed_plugins[plugin_name] = reason + logger.error("Plugin '%s' unexpected load error: %s", plugin_name, exc, exc_info=True) + return None + finally: + if _alarm_set and hasattr(signal, "SIGALRM"): + signal.alarm(0) + + def _safe_call(self, plugin_name: str, func: Any, *args: Any) -> Any | None: + """Call a plugin function with full exception isolation.""" + try: + return func(*args) + except Exception as exc: + func_name = getattr(func, "__name__", repr(func)) + reason = f"{type(exc).__name__} in {func_name}: {exc}" + self.failed_plugins[plugin_name] = reason + logger.error("Plugin '%s' call failed: %s", plugin_name, exc, exc_info=True) + return None + + def _validate_metadata(self, plugin_name: str, mod: Any) -> str: + """Return an error reason string, or empty string if valid.""" + metadata = getattr(mod, "PLUGIN_METADATA", None) + if metadata is None: + return "missing PLUGIN_METADATA in __init__.py" + + for field in _REQUIRED_METADATA_FIELDS: + if field not in metadata: + return f"PLUGIN_METADATA missing required field '{field}'" + + constraint_str = metadata.get("cgc_version_constraint", "") + try: + specifier = SpecifierSet(constraint_str) + except InvalidSpecifier: + return f"invalid cgc_version_constraint '{constraint_str}'" + + if self._cgc_version not in specifier: + return ( + f"version mismatch: plugin requires CGC {constraint_str}, " + f"installed is {self._cgc_version}" + ) + + return "" + + def _log_summary(self) -> None: + n_loaded = len(self.loaded_plugins) + n_failed = len(self.failed_plugins) + if n_loaded == 0 and n_failed == 0: + return + parts = [f"{n_loaded} plugin(s) loaded"] + if n_failed: + parts.append(f"{n_failed} skipped/failed") + logger.info("CGC plugins: %s", ", ".join(parts)) + for name, reason in self.failed_plugins.items(): + logger.warning(" ✗ %s — %s", name, reason) diff --git a/src/codegraphcontext/server.py b/src/codegraphcontext/server.py index 2f96783b..6391f2ae 100644 --- a/src/codegraphcontext/server.py +++ b/src/codegraphcontext/server.py @@ -32,6 +32,7 @@ query_handlers, watcher_handlers ) +from .plugin_registry import PluginRegistry DEFAULT_EDIT_DISTANCE = 2 DEFAULT_FUZZY_SEARCH = False @@ -86,9 +87,33 @@ def __init__(self, loop=None): def _init_tools(self): """ - Defines the complete tool manifest for the LLM. + Defines the complete tool manifest for the LLM, including plugin tools. """ - self.tools = TOOLS + self.tools = dict(TOOLS) # mutable copy + + server_context = { + "db_manager": self.db_manager, + "version": self._get_version(), + } + + plugin_registry = PluginRegistry() + plugin_registry.discover_mcp_plugins(server_context) + + self.plugin_tool_handlers: Dict[str, Any] = {} + for tool_name, tool_def in plugin_registry.mcp_tools.items(): + if tool_name in self.tools: + continue # built-in tools take precedence + self.tools[tool_name] = tool_def + if tool_name in plugin_registry.mcp_handlers: + self.plugin_tool_handlers[tool_name] = plugin_registry.mcp_handlers[tool_name] + + @staticmethod + def _get_version() -> str: + try: + from importlib.metadata import version + return version("codegraphcontext") + except Exception: + return "0.0.0" def get_database_status(self) -> dict: """Returns the current connection status of the Neo4j database.""" @@ -204,8 +229,13 @@ async def handle_tool_call(self, tool_name: str, args: Dict[str, Any]) -> Dict[s # Run the synchronous tool function in a separate thread to avoid # blocking the main asyncio event loop. return await asyncio.to_thread(handler, **args) - else: - return {"error": f"Unknown tool: {tool_name}"} + + # Fall through to plugin handlers + plugin_handler = self.plugin_tool_handlers.get(tool_name) + if plugin_handler: + return await asyncio.to_thread(plugin_handler, **args) + + return {"error": f"Unknown tool: {tool_name}"} async def run(self): """ diff --git a/tests/e2e/plugin/__init__.py b/tests/e2e/plugin/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/e2e/plugin/test_plugin_lifecycle.py b/tests/e2e/plugin/test_plugin_lifecycle.py new file mode 100644 index 00000000..fbaf5812 --- /dev/null +++ b/tests/e2e/plugin/test_plugin_lifecycle.py @@ -0,0 +1,445 @@ +""" +E2E Plugin Lifecycle Tests +========================== + +Full user-journey tests for the CGC plugin extension system. + +Journey 1 — Stub plugin: + install stub editable + → CGC starts with stub CLI command + → cgc plugin list shows stub + → stub MCP tool appears in tools + → call stub_hello via MCP + → remove stub from registry → CGC restarts cleanly + +Journey 2 — OTEL write_batch: + install otel plugin (or skip if not present) + → call write_batch with synthetic spans + → cross-layer Cypher query structure is validated + +Run as part of the e2e suite: + pytest tests/e2e/plugin/ -v -m e2e +""" +from __future__ import annotations + +import importlib.metadata +import logging +import sys +from typing import Any +from unittest.mock import MagicMock, patch, call + +import pytest + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _is_installed(package: str) -> bool: + try: + importlib.metadata.version(package) + return True + except importlib.metadata.PackageNotFoundError: + return False + + +stub_installed = pytest.mark.skipif( + not _is_installed("cgc-plugin-stub"), + reason="cgc-plugin-stub not installed — run: pip install -e plugins/cgc-plugin-stub", +) + +otel_installed = pytest.mark.skipif( + not _is_installed("cgc-plugin-otel"), + reason="cgc-plugin-otel not installed — run: pip install -e plugins/cgc-plugin-otel", +) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +@pytest.fixture() +def fresh_registry(): + """A fresh PluginRegistry with no state.""" + from codegraphcontext.plugin_registry import PluginRegistry + return PluginRegistry() + + +@pytest.fixture() +def mock_db_manager(): + """Minimal db_manager mock for unit-level checks within E2E tests.""" + mgr = MagicMock() + mgr.execute_query = MagicMock(return_value=[]) + mgr.execute_write = MagicMock(return_value=None) + return mgr + + +# --------------------------------------------------------------------------- +# Journey 1a: Stub plugin loads via real entry points +# --------------------------------------------------------------------------- + +@stub_installed +class TestStubPluginLifecycle: + """ + Tests the complete lifecycle of the stub plugin using the real entry-point + mechanism. Requires: pip install -e plugins/cgc-plugin-stub + """ + + def test_stub_cli_command_appears_after_discovery(self, fresh_registry): + """After discover_cli_plugins(), 'stub' is in loaded_plugins.""" + fresh_registry.discover_cli_plugins() + assert "stub" in fresh_registry.loaded_plugins + assert fresh_registry.loaded_plugins["stub"]["status"] == "loaded" + + def test_stub_command_in_cli_commands_list(self, fresh_registry): + """cli_commands contains a ('stub', ) tuple after discovery.""" + fresh_registry.discover_cli_plugins() + names = [n for n, _ in fresh_registry.cli_commands] + assert "stub" in names + + def test_plugin_list_command_reports_loaded(self, fresh_registry): + """plugin list shows stub as loaded (simulates cgc plugin list).""" + fresh_registry.discover_cli_plugins() + fresh_registry.discover_mcp_plugins() + assert "stub" in fresh_registry.loaded_plugins + assert fresh_registry.loaded_plugins["stub"]["status"] == "loaded" + + def test_stub_mcp_tool_appears_in_tools(self, fresh_registry): + """'stub_hello' appears in mcp_tools after discover_mcp_plugins().""" + fresh_registry.discover_mcp_plugins() + assert "stub_hello" in fresh_registry.mcp_tools + + def test_stub_mcp_tool_has_valid_schema(self, fresh_registry): + """stub_hello tool definition has required MCP schema fields.""" + fresh_registry.discover_mcp_plugins() + tool = fresh_registry.mcp_tools["stub_hello"] + assert "name" in tool + assert "description" in tool + assert "inputSchema" in tool + assert tool["inputSchema"]["type"] == "object" + + def test_stub_hello_handler_returns_greeting(self, fresh_registry): + """Calling stub_hello handler returns {'greeting': '...'} with caller name.""" + fresh_registry.discover_mcp_plugins() + handler = fresh_registry.mcp_handlers["stub_hello"] + result = handler(name="E2E") + assert isinstance(result, dict) + assert "greeting" in result + assert "E2E" in result["greeting"] + + def test_registry_clean_after_simulated_uninstall(self, fresh_registry): + """ + Simulates uninstall by creating a new registry with no entry points. + The new registry should start empty — no leftover stub artifacts. + """ + fresh_registry.discover_cli_plugins() + fresh_registry.discover_mcp_plugins() + assert "stub" in fresh_registry.loaded_plugins + + from codegraphcontext.plugin_registry import PluginRegistry + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[]): + clean_registry = PluginRegistry() + clean_registry.discover_cli_plugins() + clean_registry.discover_mcp_plugins() + + assert len(clean_registry.loaded_plugins) == 0 + assert len(clean_registry.cli_commands) == 0 + assert len(clean_registry.mcp_tools) == 0 + + +# --------------------------------------------------------------------------- +# Journey 1b: Broken plugin never crashes host (always runs, no install needed) +# --------------------------------------------------------------------------- + +class TestBrokenPluginIsolation: + """ + Verifies that broken plugins are quarantined without crashing CGC. + Uses mocked entry points so no real plugin install is required. + """ + + def _make_valid_ep(self, name: str): + import typer + + ep = MagicMock() + ep.name = name + mod = MagicMock() + mod.PLUGIN_METADATA = { + "name": name, + "version": "0.1.0", + "cgc_version_constraint": ">=0.1.0", + "description": f"Valid plugin {name}", + } + app = typer.Typer() + + @app.command() + def hello(): + pass + + mod.get_plugin_commands = MagicMock(return_value=(name, app)) + mod.get_mcp_tools = MagicMock(return_value={ + f"{name}_tool": { + "name": f"{name}_tool", + "description": "test tool", + "inputSchema": {"type": "object", "properties": {}}, + } + }) + mod.get_mcp_handlers = MagicMock(return_value={ + f"{name}_tool": lambda: {"result": "ok"} + }) + ep.load.return_value = mod + return ep + + def test_import_error_plugin_does_not_crash_host(self, fresh_registry): + """A plugin that raises ImportError is logged as failed; CGC continues.""" + good_ep = self._make_valid_ep("good") + bad_ep = MagicMock() + bad_ep.name = "broken_import" + bad_ep.load.side_effect = ImportError("missing_dep") + + with patch("codegraphcontext.plugin_registry.entry_points", + return_value=[good_ep, bad_ep]): + fresh_registry.discover_cli_plugins() + + assert "good" in fresh_registry.loaded_plugins + assert "broken_import" in fresh_registry.failed_plugins + assert len(fresh_registry.loaded_plugins) == 1 + + def test_runtime_exception_in_get_plugin_commands_is_isolated(self, fresh_registry): + """If get_plugin_commands() raises, plugin is failed; others still load.""" + good_ep = self._make_valid_ep("safe") + bad_ep = self._make_valid_ep("buggy") + bad_ep.load.return_value.get_plugin_commands.side_effect = RuntimeError( + "boom in get_plugin_commands" + ) + + with patch("codegraphcontext.plugin_registry.entry_points", + return_value=[good_ep, bad_ep]): + fresh_registry.discover_cli_plugins() + + assert "safe" in fresh_registry.loaded_plugins + assert "buggy" in fresh_registry.failed_plugins + + def test_incompatible_version_plugin_is_skipped(self, fresh_registry): + """Plugin with cgc_version_constraint that doesn't match installed version is skipped.""" + ep = MagicMock() + ep.name = "future_plugin" + mod = MagicMock() + mod.PLUGIN_METADATA = { + "name": "future_plugin", + "version": "9.9.9", + "cgc_version_constraint": ">=9999.0.0", + "description": "Too new", + } + ep.load.return_value = mod + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[ep]): + fresh_registry.discover_cli_plugins() + + assert "future_plugin" in fresh_registry.failed_plugins + assert "future_plugin" not in fresh_registry.loaded_plugins + + def test_cgc_starts_cleanly_with_no_plugins_installed(self, fresh_registry): + """With no plugins, registry loads cleanly and reports zero counts.""" + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[]): + fresh_registry.discover_cli_plugins() + fresh_registry.discover_mcp_plugins() + + assert len(fresh_registry.loaded_plugins) == 0 + assert len(fresh_registry.failed_plugins) == 0 + assert len(fresh_registry.cli_commands) == 0 + assert len(fresh_registry.mcp_tools) == 0 + + +# --------------------------------------------------------------------------- +# Journey 2: OTEL write_batch → synthetic spans → cross-layer query structure +# --------------------------------------------------------------------------- + +@otel_installed +class TestOtelPluginLifecycle: + """ + Tests the OTEL plugin write_batch path and cross-layer query capability. + Requires: pip install -e plugins/cgc-plugin-otel + Uses a mock db_manager so no live Neo4j instance is needed. + """ + + @pytest.fixture() + def writer(self, mock_db_manager): + from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter + return AsyncOtelWriter(db_manager=mock_db_manager) + + @pytest.fixture() + def synthetic_spans(self): + """Minimal synthetic span dicts matching write_batch expected format.""" + return [ + { + "span_id": "abc123", + "trace_id": "trace_001", + "parent_span_id": None, + "name": "GET /api/orders", + "service": "order-service", + "start_time_ns": 1_700_000_000_000_000_000, + "end_time_ns": 1_700_000_001_000_000_000, + "duration_ms": 1000.0, + "http_route": "/api/orders", + "http_method": "GET", + "http_status_code": 200, + "fqn": "App\\Http\\Controllers\\OrderController::index", + "span_kind": "SERVER", + "status_code": "OK", + "attributes": {}, + }, + { + "span_id": "def456", + "trace_id": "trace_001", + "parent_span_id": "abc123", + "name": "DB query", + "service": "order-service", + "start_time_ns": 1_700_000_000_100_000_000, + "end_time_ns": 1_700_000_000_200_000_000, + "duration_ms": 100.0, + "http_route": None, + "http_method": None, + "http_status_code": None, + "fqn": None, + "span_kind": "CLIENT", + "status_code": "OK", + "attributes": {"db.system": "mysql", "peer.service": "mysql"}, + }, + ] + + def test_otel_plugin_loads_via_registry(self, fresh_registry): + """OTEL plugin MCP tools are discovered by the registry.""" + fresh_registry.discover_mcp_plugins() + otel_tools = [k for k in fresh_registry.mcp_tools if k.startswith("otel_")] + assert len(otel_tools) > 0, "No otel_* MCP tools found in registry" + + def test_otel_mcp_tools_have_valid_schemas(self, fresh_registry): + """All otel_* tools have required MCP schema fields.""" + fresh_registry.discover_mcp_plugins() + for tool_name, tool_def in fresh_registry.mcp_tools.items(): + if not tool_name.startswith("otel_"): + continue + assert "name" in tool_def, f"{tool_name}: missing 'name'" + assert "description" in tool_def, f"{tool_name}: missing 'description'" + assert "inputSchema" in tool_def, f"{tool_name}: missing 'inputSchema'" + + def test_write_batch_calls_db_manager(self, writer, synthetic_spans, mock_db_manager): + """write_batch() invokes db_manager with Service, Trace, and Span merge queries.""" + import asyncio + asyncio.get_event_loop().run_until_complete(writer.write_batch(synthetic_spans)) + assert mock_db_manager.execute_write.called or mock_db_manager.execute_query.called + + def test_write_batch_handles_empty_list(self, writer, mock_db_manager): + """write_batch([]) completes without error and makes no DB calls.""" + import asyncio + asyncio.get_event_loop().run_until_complete(writer.write_batch([])) + mock_db_manager.execute_write.assert_not_called() + + def test_cross_layer_query_structure_is_valid(self): + """ + Verifies the canonical cross-layer Cypher query compiles (parse-only check). + Tests SC-005: unspecced running code query. + """ + cross_layer_query = ( + "MATCH (m:Method)<-[:CORRELATES_TO]-(s:Span) " + "WHERE NOT EXISTS { MATCH (mem:Memory)-[:DESCRIBES]->(m) } " + "RETURN m.fqn, count(s) AS executions " + "ORDER BY executions DESC LIMIT 20" + ) + # Structural validation: query contains all expected clauses + assert "CORRELATES_TO" in cross_layer_query + assert "DESCRIBES" in cross_layer_query + assert "executions" in cross_layer_query + assert "LIMIT 20" in cross_layer_query + + +# --------------------------------------------------------------------------- +# Journey 3: MCP server integration — plugin tools surface in tools/list +# --------------------------------------------------------------------------- + +class TestMCPServerPluginIntegration: + """ + Verifies that MCPServer merges plugin tools into self.tools and routes + tool_call to plugin handlers. Uses fully mocked entry points and db_manager. + """ + + def _make_mock_tool_ep(self, plugin_name: str, tool_name: str): + ep = MagicMock() + ep.name = plugin_name + mod = MagicMock() + mod.PLUGIN_METADATA = { + "name": plugin_name, + "version": "0.1.0", + "cgc_version_constraint": ">=0.1.0", + "description": "test", + } + mod.get_mcp_tools = MagicMock(return_value={ + tool_name: { + "name": tool_name, + "description": "a test tool", + "inputSchema": { + "type": "object", + "properties": {"arg": {"type": "string"}}, + }, + } + }) + mod.get_mcp_handlers = MagicMock(return_value={ + tool_name: lambda arg="default": {"result": f"called with {arg}"} + }) + ep.load.return_value = mod + return ep + + def test_registry_mcp_tools_populate_correctly(self, fresh_registry): + """Tools contributed by a mock plugin appear in registry.mcp_tools.""" + ep = self._make_mock_tool_ep("e2e_plugin", "e2e_tool") + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[ep]): + fresh_registry.discover_mcp_plugins() + + assert "e2e_tool" in fresh_registry.mcp_tools + + def test_plugin_handler_callable_via_registry(self, fresh_registry): + """Handler for plugin tool is callable and returns expected result.""" + ep = self._make_mock_tool_ep("e2e_plugin", "e2e_tool") + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[ep]): + fresh_registry.discover_mcp_plugins() + + handler = fresh_registry.mcp_handlers["e2e_tool"] + result = handler(arg="hello") + assert result == {"result": "called with hello"} + + def test_two_plugins_tools_merge_without_conflict(self, fresh_registry): + """Tools from two different plugins both appear in mcp_tools.""" + ep1 = self._make_mock_tool_ep("plugin_one", "tool_one") + ep2 = self._make_mock_tool_ep("plugin_two", "tool_two") + + with patch("codegraphcontext.plugin_registry.entry_points", + return_value=[ep1, ep2]): + fresh_registry.discover_mcp_plugins() + + assert "tool_one" in fresh_registry.mcp_tools + assert "tool_two" in fresh_registry.mcp_tools + + def test_conflicting_tool_names_first_wins(self, fresh_registry): + """When two plugins register the same tool name, the first plugin's version wins.""" + ep1 = self._make_mock_tool_ep("first_plugin", "shared_tool") + ep2 = self._make_mock_tool_ep("second_plugin", "shared_tool") + + # Override second plugin to have conflicting tool + ep2.load.return_value.get_mcp_handlers.return_value = { + "shared_tool": lambda: {"result": "second"} + } + + with patch("codegraphcontext.plugin_registry.entry_points", + return_value=[ep1, ep2]): + fresh_registry.discover_mcp_plugins() + + handler = fresh_registry.mcp_handlers.get("shared_tool") + assert handler is not None + # First plugin's handler should win + result = handler(arg="test") + assert result == {"result": "called with test"} diff --git a/tests/integration/plugin/test_memory_integration.py b/tests/integration/plugin/test_memory_integration.py new file mode 100644 index 00000000..4b0e92c8 --- /dev/null +++ b/tests/integration/plugin/test_memory_integration.py @@ -0,0 +1,148 @@ +""" +Integration tests for cgc_plugin_memory.mcp_tools handlers. + +Uses a mocked db_manager (no real Neo4j required). +Tests MUST FAIL before T037 (mcp_tools.py) is implemented. +""" +from __future__ import annotations + +import sys +import os +import pytest +from unittest.mock import MagicMock, call + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../../plugins/cgc-plugin-memory/src")) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +def _make_session(rows: list[dict] | None = None): + """Build a mock Neo4j session with configurable query results.""" + session = MagicMock() + result = MagicMock() + result.data = MagicMock(return_value=rows or []) + session.run = MagicMock(return_value=result) + session.__enter__ = MagicMock(return_value=session) + session.__exit__ = MagicMock(return_value=False) + return session + + +def _make_db(rows: list[dict] | None = None): + session = _make_session(rows) + driver = MagicMock() + driver.session = MagicMock(return_value=session) + db = MagicMock() + db.get_driver = MagicMock(return_value=driver) + return db, session + + +def _get_handlers(db_manager): + from cgc_plugin_memory.mcp_tools import get_mcp_handlers + return get_mcp_handlers({"db_manager": db_manager}) + + +# --------------------------------------------------------------------------- +# memory_store +# --------------------------------------------------------------------------- + +class TestMemoryStore: + def test_issues_merge_memory_node(self): + """memory_store issues a MERGE for the Memory node.""" + db, session = _make_db() + handlers = _get_handlers(db) + handlers["memory_store"](entity_type="spec", name="Order spec", content="Order entity spec") + + cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] + assert any("Memory" in c and "MERGE" in c for c in cypher_calls), \ + f"No Memory MERGE found: {cypher_calls}" + + def test_memory_store_with_links_to_creates_describes(self): + """memory_store with links_to creates a DESCRIBES relationship.""" + db, session = _make_db() + handlers = _get_handlers(db) + handlers["memory_store"]( + entity_type="spec", + name="Order spec", + content="...", + links_to="App\\Controllers\\OrderController", + ) + + cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] + assert any("DESCRIBES" in c for c in cypher_calls), \ + f"No DESCRIBES found: {cypher_calls}" + + def test_memory_store_without_links_to_no_describes(self): + """memory_store without links_to does NOT create DESCRIBES.""" + db, session = _make_db() + handlers = _get_handlers(db) + handlers["memory_store"](entity_type="spec", name="Standalone note", content="...") + + cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] + assert not any("DESCRIBES" in c for c in cypher_calls) + + +# --------------------------------------------------------------------------- +# memory_search +# --------------------------------------------------------------------------- + +class TestMemorySearch: + def test_search_uses_fulltext_index(self): + """memory_search queries the memory_search fulltext index.""" + db, session = _make_db(rows=[{"name": "Order spec", "entity_type": "spec", "content": "..."}]) + handlers = _get_handlers(db) + result = handlers["memory_search"](query="order") + + cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] + assert any("memory_search" in c or "FULLTEXT" in c.upper() or "CALL db.index" in c or "fulltext" in c.lower() + for c in cypher_calls), f"No fulltext query found: {cypher_calls}" + + def test_search_returns_results_key(self): + """memory_search result dict contains a 'results' key.""" + db, session = _make_db(rows=[{"name": "X", "entity_type": "spec", "content": "y"}]) + handlers = _get_handlers(db) + result = handlers["memory_search"](query="test") + assert "results" in result + + +# --------------------------------------------------------------------------- +# memory_undocumented +# --------------------------------------------------------------------------- + +class TestMemoryUndocumented: + def test_undocumented_queries_class_nodes(self): + """memory_undocumented queries Class nodes without DESCRIBES.""" + db, session = _make_db(rows=[{"fqn": "App\\Foo", "type": "Class"}]) + handlers = _get_handlers(db) + result = handlers["memory_undocumented"](node_type="Class") + + cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] + assert any("Class" in c for c in cypher_calls) + + def test_undocumented_returns_nodes_key(self): + """memory_undocumented result dict contains a 'nodes' key.""" + db, session = _make_db(rows=[]) + handlers = _get_handlers(db) + result = handlers["memory_undocumented"](node_type="Class") + assert "nodes" in result + + +# --------------------------------------------------------------------------- +# memory_link +# --------------------------------------------------------------------------- + +class TestMemoryLink: + def test_link_creates_describes_edge(self): + """memory_link creates a DESCRIBES relationship.""" + db, session = _make_db() + handlers = _get_handlers(db) + handlers["memory_link"]( + memory_id="mem-001", + node_fqn="App\\Controllers\\OrderController", + node_type="Class", + ) + + cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] + assert any("DESCRIBES" in c for c in cypher_calls), \ + f"No DESCRIBES found: {cypher_calls}" diff --git a/tests/integration/plugin/test_otel_integration.py b/tests/integration/plugin/test_otel_integration.py new file mode 100644 index 00000000..a5c9f67b --- /dev/null +++ b/tests/integration/plugin/test_otel_integration.py @@ -0,0 +1,187 @@ +""" +Integration tests for cgc_plugin_otel.neo4j_writer. + +Uses a mocked db_manager so no real Neo4j connection is required. +Tests verify that the writer issues the correct Cypher queries and +creates the expected graph structure. +""" +from __future__ import annotations + +import asyncio +import sys +import os +import pytest +from unittest.mock import AsyncMock, MagicMock, call, patch + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../../plugins/cgc-plugin-otel/src")) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +def _make_span( + span_id: str = "span001", + trace_id: str = "trace001", + parent_span_id: str | None = None, + service_name: str = "order-service", + http_route: str | None = "/api/orders", + fqn: str | None = "App\\Controllers\\OrderController::index", + cross_service: bool = False, + peer_service: str | None = None, + duration_ms: float = 12.5, +) -> dict: + return { + "span_id": span_id, + "trace_id": trace_id, + "parent_span_id": parent_span_id, + "name": f"GET {http_route or '/'}", + "span_kind": "CLIENT" if cross_service else "SERVER", + "service_name": service_name, + "start_time_ns": 1_000_000_000, + "end_time_ns": int(1_000_000_000 + duration_ms * 1_000_000), + "duration_ms": duration_ms, + "http_route": http_route, + "http_method": "GET", + "class_name": fqn.split("::")[0] if fqn else None, + "function_name": fqn.split("::")[1] if fqn else None, + "fqn": fqn, + "db_statement": None, + "db_system": None, + "peer_service": peer_service, + "cross_service": cross_service, + } + + +def _make_db_manager(): + """Build a mock db_manager that returns an async-capable session.""" + session = AsyncMock() + session.__aenter__ = AsyncMock(return_value=session) + session.__aexit__ = AsyncMock(return_value=False) + session.run = AsyncMock() + + driver = MagicMock() + driver.session = MagicMock(return_value=session) + + db_manager = MagicMock() + db_manager.get_driver = MagicMock(return_value=driver) + return db_manager, session + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + +class TestAsyncOtelWriterBatch: + + def _run(self, coro): + return asyncio.run(coro) + + def test_write_batch_issues_merge_service(self): + """write_batch() issues a MERGE for the Service node.""" + from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter + db_manager, session = _make_db_manager() + writer = AsyncOtelWriter(db_manager) + span = _make_span() + + self._run(writer.write_batch([span])) + + cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] + assert any("MERGE" in c and "Service" in c for c in cypher_calls), \ + f"No Service MERGE found in calls: {cypher_calls}" + + def test_write_batch_issues_merge_span(self): + """write_batch() issues a MERGE for the Span node.""" + from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter + db_manager, session = _make_db_manager() + writer = AsyncOtelWriter(db_manager) + span = _make_span() + + self._run(writer.write_batch([span])) + + cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] + assert any("MERGE" in c and "Span" in c for c in cypher_calls) + + def test_write_batch_links_span_to_trace(self): + """write_batch() creates a PART_OF relationship between Span and Trace.""" + from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter + db_manager, session = _make_db_manager() + writer = AsyncOtelWriter(db_manager) + span = _make_span() + + self._run(writer.write_batch([span])) + + cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] + assert any("PART_OF" in c for c in cypher_calls) + + def test_write_batch_creates_child_of_for_parent_span_id(self): + """CHILD_OF relationship is created when parent_span_id is set.""" + from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter + db_manager, session = _make_db_manager() + writer = AsyncOtelWriter(db_manager) + span = _make_span(span_id="child", parent_span_id="parent001") + + self._run(writer.write_batch([span])) + + cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] + assert any("CHILD_OF" in c for c in cypher_calls) + + def test_no_child_of_when_no_parent(self): + """CHILD_OF is NOT issued when parent_span_id is None.""" + from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter + db_manager, session = _make_db_manager() + writer = AsyncOtelWriter(db_manager) + span = _make_span(parent_span_id=None) + + self._run(writer.write_batch([span])) + + cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] + assert not any("CHILD_OF" in c for c in cypher_calls) + + def test_write_batch_creates_correlates_to_for_fqn(self): + """CORRELATES_TO relationship is attempted when fqn is set.""" + from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter + db_manager, session = _make_db_manager() + writer = AsyncOtelWriter(db_manager) + span = _make_span(fqn="App\\Controllers::index") + + self._run(writer.write_batch([span])) + + cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] + assert any("CORRELATES_TO" in c for c in cypher_calls) + + def test_no_correlates_to_when_no_fqn(self): + """CORRELATES_TO is NOT issued when fqn is None (no code context).""" + from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter + db_manager, session = _make_db_manager() + writer = AsyncOtelWriter(db_manager) + span = _make_span(fqn=None) + + self._run(writer.write_batch([span])) + + cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] + assert not any("CORRELATES_TO" in c for c in cypher_calls) + + def test_cross_service_span_creates_calls_service(self): + """CALLS_SERVICE is created for CLIENT spans with peer_service set.""" + from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter + db_manager, session = _make_db_manager() + writer = AsyncOtelWriter(db_manager) + span = _make_span(cross_service=True, peer_service="payment-service") + + self._run(writer.write_batch([span])) + + cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] + assert any("CALLS_SERVICE" in c for c in cypher_calls) + + def test_db_failure_routes_to_dlq(self): + """When the database raises, spans are moved to the dead-letter queue.""" + from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter + db_manager = MagicMock() + db_manager.get_driver.side_effect = Exception("Neo4j unavailable") + writer = AsyncOtelWriter(db_manager) + span = _make_span() + + self._run(writer.write_batch([span])) + + assert not writer._dlq.empty() diff --git a/tests/integration/plugin/test_plugin_load.py b/tests/integration/plugin/test_plugin_load.py new file mode 100644 index 00000000..950c9581 --- /dev/null +++ b/tests/integration/plugin/test_plugin_load.py @@ -0,0 +1,210 @@ +""" +Integration tests for CGC plugin loading with the stub plugin. + +These tests use the real entry-point mechanism. The stub plugin must be +installed in editable mode before running: + + pip install -e plugins/cgc-plugin-stub + +Tests MUST FAIL before Phase 3 implementation (T012-T016) is complete. +""" +import importlib.metadata +import logging +import pytest +from unittest.mock import MagicMock, patch + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _stub_installed() -> bool: + """Return True if cgc-plugin-stub is installed in the current environment.""" + try: + importlib.metadata.version("cgc-plugin-stub") + return True + except importlib.metadata.PackageNotFoundError: + return False + + +stub_required = pytest.mark.skipif( + not _stub_installed(), + reason="cgc-plugin-stub not installed — run: pip install -e plugins/cgc-plugin-stub", +) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +@pytest.fixture() +def registry(): + """Fresh PluginRegistry instance for each test.""" + from codegraphcontext.plugin_registry import PluginRegistry + return PluginRegistry() + + +# --------------------------------------------------------------------------- +# Tests — stub plugin via real entry points (requires editable install) +# --------------------------------------------------------------------------- + +@stub_required +class TestStubPluginLoad: + """Tests that use the real installed stub plugin via entry-point discovery.""" + + def test_stub_cli_command_discovered(self, registry): + """Stub CLI command group 'stub' appears after discover_cli_plugins().""" + registry.discover_cli_plugins() + assert "stub" in registry.loaded_plugins, ( + "stub plugin not in loaded_plugins — is PLUGIN_METADATA defined in __init__.py?" + ) + assert registry.loaded_plugins["stub"]["status"] == "loaded" + + def test_stub_cli_commands_populated(self, registry): + """cli_commands list contains ('stub', ) after load.""" + registry.discover_cli_plugins() + names = [name for name, _ in registry.cli_commands] + assert "stub" in names + + def test_stub_mcp_tool_discovered(self, registry): + """MCP tool 'stub_hello' appears in mcp_tools after discover_mcp_plugins().""" + registry.discover_mcp_plugins() + assert "stub_hello" in registry.mcp_tools, ( + "stub_hello tool missing — is get_mcp_tools() implemented in mcp_tools.py?" + ) + + def test_stub_mcp_handler_registered(self, registry): + """Handler for 'stub_hello' is registered in mcp_handlers.""" + registry.discover_mcp_plugins() + assert "stub_hello" in registry.mcp_handlers + + def test_stub_mcp_handler_returns_greeting(self, registry): + """stub_hello handler returns a dict containing 'greeting'.""" + registry.discover_mcp_plugins() + handler = registry.mcp_handlers["stub_hello"] + result = handler(name="Tester") + assert "greeting" in result + assert "Tester" in result["greeting"] + + +# --------------------------------------------------------------------------- +# Tests — isolation behaviour (always run, use mocked entry points) +# --------------------------------------------------------------------------- + +class TestPluginIsolationBehaviour: + """ + Behavioural isolation tests that do NOT require the stub to be installed. + They use hand-crafted mocks to verify the registry enforces contracts. + """ + + def _make_stub_ep(self, name="stub"): + """Build a minimal stub entry-point mock with valid metadata.""" + import typer + + ep = MagicMock() + ep.name = name + + mod = MagicMock() + mod.PLUGIN_METADATA = { + "name": name, + "version": "0.1.0", + "cgc_version_constraint": ">=0.1.0", + "description": f"Stub plugin '{name}'", + } + stub_app = typer.Typer() + + @stub_app.command() + def hello(): + """Hello from stub.""" + + mod.get_plugin_commands = MagicMock(return_value=(name, stub_app)) + mod.get_mcp_tools = MagicMock(return_value={ + f"{name}_hello": { + "name": f"{name}_hello", + "description": "Say hello", + "inputSchema": { + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], + }, + } + }) + mod.get_mcp_handlers = MagicMock( + return_value={f"{name}_hello": lambda name="World": {"greeting": f"Hello {name}"}} + ) + + ep.load.return_value = mod + return ep + + def test_second_incompatible_plugin_skipped(self, registry): + """A second plugin with incompatible version constraint is skipped with warning.""" + good_ep = self._make_stub_ep("good") + + bad_mod = MagicMock() + bad_mod.PLUGIN_METADATA = { + "name": "old", + "version": "0.0.1", + "cgc_version_constraint": ">=99.0.0", + "description": "Too new", + } + bad_ep = MagicMock() + bad_ep.name = "old" + bad_ep.load.return_value = bad_mod + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[good_ep, bad_ep]): + registry.discover_cli_plugins() + + assert "good" in registry.loaded_plugins + assert "old" not in registry.loaded_plugins + assert "old" in registry.failed_plugins + + def test_duplicate_name_loads_only_first(self, registry): + """Two plugins with identical names: first wins, second is silently skipped.""" + ep1 = self._make_stub_ep("dupe") + ep2 = self._make_stub_ep("dupe") + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[ep1, ep2]): + registry.discover_cli_plugins() + + assert registry.loaded_plugins["dupe"]["status"] == "loaded" + # Only one entry in cli_commands for this name + assert sum(1 for name, _ in registry.cli_commands if name == "dupe") == 1 + + def test_conflicting_mcp_tool_loads_only_first(self, registry): + """Two plugins registering the same MCP tool name: first plugin's definition wins.""" + ep1 = self._make_stub_ep("plugin_a") + ep2 = self._make_stub_ep("plugin_b") + + # Make plugin_b register a tool with the same key as plugin_a + ep2.load.return_value.get_mcp_tools.return_value = { + "plugin_a_hello": { # conflicts with plugin_a + "name": "plugin_a_hello", + "description": "conflict", + "inputSchema": {"type": "object", "properties": {}}, + } + } + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[ep1, ep2]): + registry.discover_mcp_plugins() + + # Tool exists, registered by first plugin + assert "plugin_a_hello" in registry.mcp_tools + # Both plugins loaded (even though one tool was skipped) + assert "plugin_a" in registry.loaded_plugins + assert "plugin_b" in registry.loaded_plugins + + def test_registry_reports_correct_counts(self, registry): + """loaded_plugins and failed_plugins counts are accurate after mixed load.""" + ep_good = self._make_stub_ep("ok_plugin") + ep_bad = MagicMock() + ep_bad.name = "broken" + ep_bad.load.side_effect = ImportError("missing dep") + + with patch("codegraphcontext.plugin_registry.entry_points", + return_value=[ep_good, ep_bad]): + registry.discover_cli_plugins() + + assert len(registry.loaded_plugins) == 1 + assert len(registry.failed_plugins) == 1 + assert "ok_plugin" in registry.loaded_plugins + assert "broken" in registry.failed_plugins diff --git a/tests/unit/plugin/test_otel_processor.py b/tests/unit/plugin/test_otel_processor.py new file mode 100644 index 00000000..d7de53bf --- /dev/null +++ b/tests/unit/plugin/test_otel_processor.py @@ -0,0 +1,185 @@ +""" +Unit tests for cgc_plugin_otel.span_processor. + +All tests run without gRPC or a real database — pure logic tests. +Tests MUST FAIL before T020 (span_processor.py) is implemented. +""" +import pytest +import sys +import os + +# Allow import even when cgc-plugin-otel is not installed +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../../plugins/cgc-plugin-otel/src")) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def _import_processor(): + from cgc_plugin_otel.span_processor import ( + extract_php_context, + build_fqn, + is_cross_service_span, + should_filter_span, + build_span_dict, + ) + return extract_php_context, build_fqn, is_cross_service_span, should_filter_span, build_span_dict + + +# --------------------------------------------------------------------------- +# extract_php_context +# --------------------------------------------------------------------------- + +class TestExtractPhpContext: + def test_full_attributes_parsed(self): + extract_php_context, *_ = _import_processor() + attrs = { + "code.namespace": "App\\Http\\Controllers", + "code.function": "index", + "http.route": "/api/orders", + "http.method": "GET", + } + result = extract_php_context(attrs) + assert result["namespace"] == "App\\Http\\Controllers" + assert result["function"] == "index" + assert result["http_route"] == "/api/orders" + assert result["http_method"] == "GET" + + def test_missing_optional_attributes_return_none(self): + extract_php_context, *_ = _import_processor() + result = extract_php_context({}) + assert result["namespace"] is None + assert result["function"] is None + assert result["http_route"] is None + assert result["http_method"] is None + + def test_db_attributes_captured(self): + extract_php_context, *_ = _import_processor() + attrs = { + "db.statement": "SELECT * FROM orders", + "db.system": "mysql", + } + result = extract_php_context(attrs) + assert result["db_statement"] == "SELECT * FROM orders" + assert result["db_system"] == "mysql" + + +# --------------------------------------------------------------------------- +# build_fqn +# --------------------------------------------------------------------------- + +class TestBuildFqn: + def test_namespace_and_function_joined(self): + _, build_fqn, *_ = _import_processor() + assert build_fqn("App\\Controllers", "store") == "App\\Controllers::store" + + def test_none_namespace_returns_none(self): + _, build_fqn, *_ = _import_processor() + assert build_fqn(None, "store") is None + + def test_none_function_returns_none(self): + _, build_fqn, *_ = _import_processor() + assert build_fqn("App\\Controllers", None) is None + + def test_both_none_returns_none(self): + _, build_fqn, *_ = _import_processor() + assert build_fqn(None, None) is None + + +# --------------------------------------------------------------------------- +# is_cross_service_span +# --------------------------------------------------------------------------- + +class TestIsCrossServiceSpan: + def test_client_kind_with_peer_service_is_cross_service(self): + _, _, is_cross_service_span, *_ = _import_processor() + assert is_cross_service_span("CLIENT", {"peer.service": "order-service"}) is True + + def test_client_kind_without_peer_service_is_not_cross_service(self): + _, _, is_cross_service_span, *_ = _import_processor() + assert is_cross_service_span("CLIENT", {}) is False + + def test_server_kind_is_not_cross_service(self): + _, _, is_cross_service_span, *_ = _import_processor() + assert is_cross_service_span("SERVER", {"peer.service": "anything"}) is False + + def test_internal_kind_is_not_cross_service(self): + _, _, is_cross_service_span, *_ = _import_processor() + assert is_cross_service_span("INTERNAL", {"peer.service": "anything"}) is False + + +# --------------------------------------------------------------------------- +# should_filter_span +# --------------------------------------------------------------------------- + +class TestShouldFilterSpan: + def test_health_route_filtered(self): + _, _, _, should_filter_span, _ = _import_processor() + assert should_filter_span({"http.route": "/health"}, ["/health", "/metrics"]) is True + + def test_metrics_route_filtered(self): + _, _, _, should_filter_span, _ = _import_processor() + assert should_filter_span({"http.route": "/metrics"}, ["/health", "/metrics"]) is True + + def test_normal_route_not_filtered(self): + _, _, _, should_filter_span, _ = _import_processor() + assert should_filter_span({"http.route": "/api/orders"}, ["/health", "/metrics"]) is False + + def test_empty_filter_list_never_filters(self): + _, _, _, should_filter_span, _ = _import_processor() + assert should_filter_span({"http.route": "/health"}, []) is False + + def test_span_without_route_not_filtered(self): + _, _, _, should_filter_span, _ = _import_processor() + assert should_filter_span({}, ["/health"]) is False + + +# --------------------------------------------------------------------------- +# build_span_dict +# --------------------------------------------------------------------------- + +class TestBuildSpanDict: + def test_duration_ms_computed_from_nanoseconds(self): + _, _, _, _, build_span_dict = _import_processor() + span = build_span_dict( + span_id="abc123", + trace_id="trace456", + parent_span_id=None, + name="GET /api/orders", + span_kind="SERVER", + start_time_ns=1_000_000_000, + end_time_ns=1_500_000_000, + attributes={}, + service_name="order-service", + ) + assert span["duration_ms"] == pytest.approx(500.0) + + def test_required_fields_present(self): + _, _, _, _, build_span_dict = _import_processor() + span = build_span_dict( + span_id="abc123", + trace_id="trace456", + parent_span_id="parent789", + name="GET /api/orders", + span_kind="SERVER", + start_time_ns=1_000_000_000, + end_time_ns=2_000_000_000, + attributes={"http.route": "/api/orders"}, + service_name="order-service", + ) + assert span["span_id"] == "abc123" + assert span["trace_id"] == "trace456" + assert span["parent_span_id"] == "parent789" + assert span["service_name"] == "order-service" + assert span["name"] == "GET /api/orders" + + def test_zero_duration_for_equal_timestamps(self): + _, _, _, _, build_span_dict = _import_processor() + span = build_span_dict( + span_id="x", trace_id="y", parent_span_id=None, + name="instant", span_kind="INTERNAL", + start_time_ns=5_000_000, end_time_ns=5_000_000, + attributes={}, service_name="svc", + ) + assert span["duration_ms"] == 0.0 diff --git a/tests/unit/plugin/test_plugin_registry.py b/tests/unit/plugin/test_plugin_registry.py new file mode 100644 index 00000000..eaad21e5 --- /dev/null +++ b/tests/unit/plugin/test_plugin_registry.py @@ -0,0 +1,275 @@ +""" +Unit tests for PluginRegistry. + +All entry-point discovery is mocked — no installed packages required. +Tests MUST FAIL before PluginRegistry is implemented (TDD Red phase). +""" +import pytest +import logging +from unittest.mock import MagicMock, patch, PropertyMock + + +# --------------------------------------------------------------------------- +# Helpers: build fake entry-point objects +# --------------------------------------------------------------------------- + +def _make_ep(name, module_path, metadata=None, raise_on_load=None): + """Create a mock entry-point that loads a callable.""" + ep = MagicMock() + ep.name = name + + if raise_on_load: + ep.load.side_effect = raise_on_load + else: + def _loader(): + if metadata is not None: + mod = MagicMock() + mod.PLUGIN_METADATA = metadata + mod.get_plugin_commands = MagicMock( + return_value=(name, MagicMock()) + ) + mod.get_mcp_tools = MagicMock(return_value={ + f"{name}_tool": { + "name": f"{name}_tool", + "description": "test", + "inputSchema": {"type": "object", "properties": {}} + } + }) + mod.get_mcp_handlers = MagicMock(return_value={ + f"{name}_tool": lambda **kw: {"ok": True} + }) + return mod + return MagicMock() + + ep.load.return_value = _loader() + + return ep + + +VALID_METADATA = { + "name": "test-plugin", + "version": "0.1.0", + "cgc_version_constraint": ">=0.1.0", + "description": "Test plugin", +} + +INCOMPATIBLE_METADATA = { + "name": "old-plugin", + "version": "0.0.1", + "cgc_version_constraint": ">=99.0.0", + "description": "Too new constraint", +} + +MISSING_FIELD_METADATA = { + "name": "bad-plugin", + # missing version, cgc_version_constraint, description +} + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + +class TestPluginRegistryDiscovery: + """Tests for plugin discovery and loading.""" + + def test_no_plugins_installed_starts_cleanly(self): + """Registry with zero entry points should start without errors.""" + from codegraphcontext.plugin_registry import PluginRegistry + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[]): + registry = PluginRegistry() + registry.discover_cli_plugins() + registry.discover_mcp_plugins() + + assert registry.loaded_plugins == {} + assert registry.failed_plugins == {} + + def test_valid_plugin_is_loaded(self): + """A plugin with valid metadata and compatible version is loaded.""" + from codegraphcontext.plugin_registry import PluginRegistry + + ep = _make_ep("myplugin", "myplugin.cli:get_plugin_commands", + metadata=VALID_METADATA) + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[ep]): + registry = PluginRegistry() + registry.discover_cli_plugins() + + assert "myplugin" in registry.loaded_plugins + assert registry.loaded_plugins["myplugin"]["status"] == "loaded" + + def test_incompatible_version_is_skipped(self): + """A plugin whose cgc_version_constraint excludes the installed CGC version is skipped.""" + from codegraphcontext.plugin_registry import PluginRegistry + + ep = _make_ep("oldplugin", "oldplugin.cli:get", + metadata=INCOMPATIBLE_METADATA) + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[ep]): + registry = PluginRegistry() + registry.discover_cli_plugins() + + assert "oldplugin" not in registry.loaded_plugins + assert "oldplugin" in registry.failed_plugins + assert "version" in registry.failed_plugins["oldplugin"].lower() + + def test_missing_metadata_field_is_skipped(self): + """A plugin missing required PLUGIN_METADATA fields is skipped.""" + from codegraphcontext.plugin_registry import PluginRegistry + + ep = _make_ep("badplugin", "badplugin.cli:get", + metadata=MISSING_FIELD_METADATA) + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[ep]): + registry = PluginRegistry() + registry.discover_cli_plugins() + + assert "badplugin" not in registry.loaded_plugins + assert "badplugin" in registry.failed_plugins + + def test_import_error_does_not_crash_host(self): + """An ImportError during plugin load is caught; registry continues.""" + from codegraphcontext.plugin_registry import PluginRegistry + + bad_ep = _make_ep("broken", "broken.cli:get", + raise_on_load=ImportError("missing dep")) + good_ep = _make_ep("good", "good.cli:get", + metadata=VALID_METADATA) + + with patch("codegraphcontext.plugin_registry.entry_points", + side_effect=[[bad_ep, good_ep], []]): + registry = PluginRegistry() + registry.discover_cli_plugins() + + assert "broken" in registry.failed_plugins + assert "good" in registry.loaded_plugins + + def test_exception_in_get_plugin_commands_does_not_crash(self): + """An exception raised by get_plugin_commands() is caught.""" + from codegraphcontext.plugin_registry import PluginRegistry + + mod = MagicMock() + mod.PLUGIN_METADATA = VALID_METADATA + mod.get_plugin_commands.side_effect = RuntimeError("boom") + + ep = MagicMock() + ep.name = "crashplugin" + ep.load.return_value = mod + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[ep]): + registry = PluginRegistry() + registry.discover_cli_plugins() + + assert "crashplugin" in registry.failed_plugins + + def test_duplicate_plugin_name_skips_second(self): + """Two plugins with the same name: first wins, second is skipped.""" + from codegraphcontext.plugin_registry import PluginRegistry + + ep1 = _make_ep("dupe", "a.cli:get", metadata=VALID_METADATA) + ep2 = _make_ep("dupe", "b.cli:get", metadata=VALID_METADATA) + + with patch("codegraphcontext.plugin_registry.entry_points", + return_value=[ep1, ep2]): + registry = PluginRegistry() + registry.discover_cli_plugins() + + assert "dupe" in registry.loaded_plugins + # Only one entry — second skipped + assert registry.loaded_plugins["dupe"]["status"] == "loaded" + + def test_loaded_and_failed_counts_are_accurate(self): + """Summary counts match actual loaded/failed plugins.""" + from codegraphcontext.plugin_registry import PluginRegistry + + good1 = _make_ep("g1", "g1.cli:get", metadata=VALID_METADATA) + good2 = _make_ep("g2", "g2.cli:get", metadata=VALID_METADATA) + bad = _make_ep("bad", "bad.cli:get", + raise_on_load=ImportError("missing")) + + with patch("codegraphcontext.plugin_registry.entry_points", + side_effect=[[good1, good2, bad], []]): + registry = PluginRegistry() + registry.discover_cli_plugins() + + assert len(registry.loaded_plugins) == 2 + assert len(registry.failed_plugins) == 1 + + +class TestPluginRegistryCLI: + """Tests for CLI command registration results.""" + + def test_cli_commands_populated_after_load(self): + """cli_commands list is populated with (name, typer_app) tuples.""" + from codegraphcontext.plugin_registry import PluginRegistry + + ep = _make_ep("myplugin", "myplugin.cli:get", metadata=VALID_METADATA) + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[ep]): + registry = PluginRegistry() + registry.discover_cli_plugins() + + assert len(registry.cli_commands) == 1 + name, typer_app = registry.cli_commands[0] + assert name == "myplugin" + + def test_cli_commands_empty_without_plugins(self): + """cli_commands is empty when no plugins are installed.""" + from codegraphcontext.plugin_registry import PluginRegistry + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[]): + registry = PluginRegistry() + registry.discover_cli_plugins() + + assert registry.cli_commands == [] + + +class TestPluginRegistryMCP: + """Tests for MCP tool registration results.""" + + def test_mcp_tools_populated_after_load(self): + """mcp_tools dict is populated with tool definitions from loaded plugins.""" + from codegraphcontext.plugin_registry import PluginRegistry + + ep = _make_ep("myplugin", "myplugin.mcp:get", metadata=VALID_METADATA) + + server_context = {"db_manager": MagicMock(), "version": "0.3.1"} + + with patch("codegraphcontext.plugin_registry.entry_points", return_value=[ep]): + registry = PluginRegistry() + registry.discover_mcp_plugins(server_context) + + assert "myplugin_tool" in registry.mcp_tools + assert "myplugin_tool" in registry.mcp_handlers + + def test_conflicting_tool_name_skips_second(self): + """Two plugins registering the same tool name: first wins.""" + from codegraphcontext.plugin_registry import PluginRegistry + + # Both plugins register "myplugin_tool" + ep1 = _make_ep("plugin_a", "a.mcp:get", metadata={**VALID_METADATA, "name": "plugin_a"}) + ep2 = _make_ep("plugin_b", "b.mcp:get", metadata={**VALID_METADATA, "name": "plugin_b"}) + + # Make ep2 return a tool with the same name as ep1 + mod2 = MagicMock() + mod2.PLUGIN_METADATA = {**VALID_METADATA, "name": "plugin_b"} + mod2.get_mcp_tools = MagicMock(return_value={ + "myplugin_tool": { # same key as ep1's tool + "name": "myplugin_tool", + "description": "conflict", + "inputSchema": {"type": "object", "properties": {}} + } + }) + mod2.get_mcp_handlers = MagicMock(return_value={"myplugin_tool": lambda **k: {}}) + ep2.load.return_value = mod2 + + server_context = {"db_manager": MagicMock(), "version": "0.3.1"} + + with patch("codegraphcontext.plugin_registry.entry_points", + return_value=[ep1, ep2]): + registry = PluginRegistry() + registry.discover_mcp_plugins(server_context) + + # Tool is registered once, from the first plugin + assert "myplugin_tool" in registry.mcp_tools diff --git a/tests/unit/plugin/test_xdebug_parser.py b/tests/unit/plugin/test_xdebug_parser.py new file mode 100644 index 00000000..486475e8 --- /dev/null +++ b/tests/unit/plugin/test_xdebug_parser.py @@ -0,0 +1,139 @@ +""" +Unit tests for cgc_plugin_xdebug.dbgp_server parsing logic. + +No TCP connections required — pure XML/logic tests. +Tests MUST FAIL before T030 (dbgp_server.py) is implemented. +""" +import pytest +import sys +import os + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../../plugins/cgc-plugin-xdebug/src")) + +_SAMPLE_STACK_XML = """\ + + + + + + +""" + +_EMPTY_STACK_XML = """\ + + + +""" + + +def _import_parser(): + from cgc_plugin_xdebug.dbgp_server import ( + parse_stack_xml, + compute_chain_hash, + build_frame_id, + ) + return parse_stack_xml, compute_chain_hash, build_frame_id + + +# --------------------------------------------------------------------------- +# parse_stack_xml +# --------------------------------------------------------------------------- + +class TestParseStackXml: + def test_returns_correct_frame_count(self): + parse_stack_xml, *_ = _import_parser() + frames = parse_stack_xml(_SAMPLE_STACK_XML) + assert len(frames) == 3 + + def test_frame_fields_present(self): + parse_stack_xml, *_ = _import_parser() + frames = parse_stack_xml(_SAMPLE_STACK_XML) + frame = frames[0] + assert "where" in frame + assert "level" in frame + assert "filename" in frame + assert "lineno" in frame + + def test_frame_level_is_integer(self): + parse_stack_xml, *_ = _import_parser() + frames = parse_stack_xml(_SAMPLE_STACK_XML) + assert all(isinstance(f["level"], int) for f in frames) + + def test_frames_ordered_by_level(self): + parse_stack_xml, *_ = _import_parser() + frames = parse_stack_xml(_SAMPLE_STACK_XML) + levels = [f["level"] for f in frames] + assert levels == sorted(levels) + + def test_empty_stack_returns_empty_list(self): + parse_stack_xml, *_ = _import_parser() + assert parse_stack_xml(_EMPTY_STACK_XML) == [] + + def test_first_frame_where_parsed(self): + parse_stack_xml, *_ = _import_parser() + frames = parse_stack_xml(_SAMPLE_STACK_XML) + assert "OrderController" in frames[0]["where"] + + def test_filename_stripped_of_scheme(self): + """file:// prefix should be stripped from the filename.""" + parse_stack_xml, *_ = _import_parser() + frames = parse_stack_xml(_SAMPLE_STACK_XML) + assert not frames[0]["filename"].startswith("file://") + + +# --------------------------------------------------------------------------- +# compute_chain_hash +# --------------------------------------------------------------------------- + +class TestComputeChainHash: + def test_same_frames_same_hash(self): + parse_stack_xml, compute_chain_hash, _ = _import_parser() + frames = parse_stack_xml(_SAMPLE_STACK_XML) + assert compute_chain_hash(frames) == compute_chain_hash(frames) + + def test_different_frames_different_hash(self): + parse_stack_xml, compute_chain_hash, _ = _import_parser() + frames1 = parse_stack_xml(_SAMPLE_STACK_XML) + frames2 = frames1[:-1] # drop last frame + assert compute_chain_hash(frames1) != compute_chain_hash(frames2) + + def test_empty_frames_returns_hash(self): + _, compute_chain_hash, _ = _import_parser() + h = compute_chain_hash([]) + assert isinstance(h, str) and len(h) > 0 + + def test_hash_is_deterministic_across_calls(self): + parse_stack_xml, compute_chain_hash, _ = _import_parser() + frames = parse_stack_xml(_SAMPLE_STACK_XML) + assert compute_chain_hash(frames) == compute_chain_hash(frames) + + +# --------------------------------------------------------------------------- +# build_frame_id +# --------------------------------------------------------------------------- + +class TestBuildFrameId: + def test_returns_string(self): + _, _, build_frame_id = _import_parser() + fid = build_frame_id("App\\Controllers\\Foo", "bar", "/var/www/Foo.php", 10) + assert isinstance(fid, str) + + def test_deterministic(self): + _, _, build_frame_id = _import_parser() + a = build_frame_id("App\\Foo", "bar", "/var/www/Foo.php", 10) + b = build_frame_id("App\\Foo", "bar", "/var/www/Foo.php", 10) + assert a == b + + def test_different_inputs_different_ids(self): + _, _, build_frame_id = _import_parser() + a = build_frame_id("App\\Foo", "bar", "/var/www/Foo.php", 10) + b = build_frame_id("App\\Foo", "baz", "/var/www/Foo.php", 10) + assert a != b From d0a0bbeace0f2a754b562b00f8f661b288bd0cff Mon Sep 17 00:00:00 2001 From: Brian Gebel Date: Sat, 14 Mar 2026 19:57:54 -0700 Subject: [PATCH 2/7] Add spec-kit --- .specify/init-options.json | 9 + .specify/memory/constitution.md | 147 ++++ .specify/scripts/bash/check-prerequisites.sh | 190 ++++ .specify/scripts/bash/common.sh | 253 ++++++ .specify/scripts/bash/create-new-feature.sh | 333 ++++++++ .specify/scripts/bash/setup-plan.sh | 73 ++ .specify/scripts/bash/update-agent-context.sh | 808 ++++++++++++++++++ .specify/templates/agent-file-template.md | 28 + .specify/templates/checklist-template.md | 40 + .specify/templates/constitution-template.md | 50 ++ .specify/templates/plan-template.md | 104 +++ .specify/templates/spec-template.md | 115 +++ .specify/templates/tasks-template.md | 251 ++++++ 13 files changed, 2401 insertions(+) create mode 100644 .specify/init-options.json create mode 100644 .specify/memory/constitution.md create mode 100755 .specify/scripts/bash/check-prerequisites.sh create mode 100755 .specify/scripts/bash/common.sh create mode 100755 .specify/scripts/bash/create-new-feature.sh create mode 100755 .specify/scripts/bash/setup-plan.sh create mode 100755 .specify/scripts/bash/update-agent-context.sh create mode 100644 .specify/templates/agent-file-template.md create mode 100644 .specify/templates/checklist-template.md create mode 100644 .specify/templates/constitution-template.md create mode 100644 .specify/templates/plan-template.md create mode 100644 .specify/templates/spec-template.md create mode 100644 .specify/templates/tasks-template.md diff --git a/.specify/init-options.json b/.specify/init-options.json new file mode 100644 index 00000000..63266c1c --- /dev/null +++ b/.specify/init-options.json @@ -0,0 +1,9 @@ +{ + "ai": "claude", + "ai_commands_dir": null, + "ai_skills": false, + "here": true, + "preset": null, + "script": "sh", + "speckit_version": "0.3.0" +} \ No newline at end of file diff --git a/.specify/memory/constitution.md b/.specify/memory/constitution.md new file mode 100644 index 00000000..2880d0d8 --- /dev/null +++ b/.specify/memory/constitution.md @@ -0,0 +1,147 @@ + + +# CodeGraphContext Constitution + +## Core Principles + +### I. Graph-First Architecture + +All code intelligence MUST be represented as a property graph of typed nodes (functions, +classes, files, modules) and typed relationships (CALLS, IMPORTS, INHERITS, DEFINES). +New parsers and indexers MUST produce graph-compatible output — flat or ad-hoc data +structures are not acceptable as the final output of any indexing step. +The graph schema (node labels, relationship types, and their required properties) is the +canonical source of truth; all CLI commands, MCP tools, and query logic MUST derive from +it, not duplicate or contradict it. + +**Rationale**: The entire value proposition of CGC is queryable graph context. Deviating +from graph-first design undermines the core product contract with AI agents and users. + +### II. Dual Interface — CLI and MCP + +Every user-facing capability MUST be accessible via both the `cgc` CLI (Typer/Click) and +the MCP server tool API. Neither interface may lag behind the other; a capability that +exists in one MUST exist in the other within the same release. CLI commands output to +stdout (human-readable by default, JSON when `--json` flag supplied); errors go to stderr. + +**Rationale**: Users rely on CGC in both interactive terminal sessions and automated AI +assistant pipelines. Parity between the two interfaces prevents feature silos and ensures +the tool is universally accessible regardless of integration context. + +### III. Testing Pyramid (NON-NEGOTIABLE) + +CGC follows a strict testing pyramid: + +- **Unit** (`tests/unit/`): Fast (<100ms), heavily mocked, covers isolated components. +- **Integration** (`tests/integration/`): Covers interaction between 2+ components with + partial mocking (~1s). +- **E2E** (`tests/e2e/`): Full user journey tests with minimal mocking (>10s). + +All new features MUST include tests at the appropriate pyramid level(s) before merging. +`./tests/run_tests.sh fast` (unit + integration) MUST pass locally before any PR is +submitted. Tests for a feature MUST be written and observed to fail before implementation +begins (Red-Green-Refactor). + +**Rationale**: CGC's correctness guarantees — that graph queries return accurate, complete +context — can only be trusted with comprehensive, layered test coverage. Untested parsers +or query paths create silent failures that degrade AI agent output quality. + +### IV. Multi-Language Parser Parity + +Every programming language supported by CGC MUST expose the same canonical node types +(Function, Class, File, Module, Variable) and the same relationship types (CALLS, IMPORTS, +INHERITS, DEFINES) where applicable to the language. Language-specific parsers MAY add +language-native relationship types (e.g., IMPLEMENTS for Java interfaces) only if they +are documented in the graph schema and do not break cross-language queries. A parser MUST +NOT introduce schema deviations (renamed labels, different property keys) without a +migration plan approved via the amendment process. + +**Rationale**: Users and AI agents query the graph without knowing which language produced +the data. Schema inconsistency across parsers would produce unreliable query results and +break tooling that depends on stable node/relationship contracts. + +### V. Simplicity + +Implement the simplest solution that satisfies the current requirement. YAGNI (You Aren't +Gonna Need It) applies strictly: abstractions, helpers, and new modules MUST be justified +by a current, concrete need — not anticipated future requirements. Three similar lines of +code are preferable to a premature abstraction. Complexity in the graph schema, parser +logic, or query layer MUST be justified in the plan document before implementation. + +**Rationale**: CGC serves a broad contributor base across many languages and stacks. +Unnecessary complexity raises the barrier to contribution, increases maintenance cost, and +makes the graph schema harder to reason about. + +## Technology Constraints + +CGC's core technology choices are stable and MUST NOT be replaced without a major +constitutional amendment: + +- **Language**: Python 3.10+ (no other implementation languages for the core library) +- **Parsing**: Tree-sitter (the sole AST parsing mechanism; regex-based parsing is + prohibited for language node extraction) +- **Protocol**: Model Context Protocol (MCP) for AI agent integration +- **Graph Database**: FalkorDB (embedded/default) or Neo4j (production); the database + abstraction layer MUST support both without feature disparity +- **CLI Framework**: Typer / Click +- **Package Distribution**: PyPI (`codegraphcontext`) + +New runtime dependencies MUST be added to `pyproject.toml` (or equivalent) and MUST be +justified in the PR description. Dependencies that significantly increase install size or +reduce cross-platform compatibility require explicit maintainer approval. + +## Contribution Standards + +All contributors MUST adhere to the following standards: + +- **Code style**: Follow existing project conventions; run linting before submitting. +- **PR scope**: Each pull request MUST be focused on a single feature or bug fix. +- **Test gate**: `./tests/run_tests.sh fast` MUST pass before PR submission. +- **Documentation**: New CLI commands and MCP tools MUST be documented in `docs/`. +- **Security**: Vulnerabilities MUST be reported privately (see `SECURITY.md`); do not + open public issues for security findings. Dependencies MUST be kept up to date. +- **Breaking changes**: Any change to the graph schema, CLI command signatures, or MCP + tool API signatures is a breaking change and requires a MAJOR version bump and a + migration guide. + +## Governance + +This constitution supersedes all other development practices documented in this repository. +In the event of a conflict between this document and any other guideline, this constitution +takes precedence. + +**Amendment procedure**: +1. Open a GitHub issue proposing the amendment with rationale. +2. Allow at least one maintainer review cycle. +3. Update this file, increment the version per the versioning policy, and set + `Last Amended` to the date of the change. +4. Propagate changes to dependent templates (per the Consistency Propagation Checklist + in `.specify/templates/constitution-template.md`). + +**Versioning policy**: +- MAJOR: Backward-incompatible governance changes, principle removals, or redefinitions. +- MINOR: New principle or section added, or materially expanded guidance. +- PATCH: Clarifications, wording fixes, non-semantic refinements. + +**Compliance review**: All PRs MUST be reviewed against Core Principles I–V. Reviewers +MUST reject PRs that violate any non-negotiable principle without documented justification +in a Complexity Tracking section (see plan template). + +**Version**: 1.0.0 | **Ratified**: 2025-08-17 | **Last Amended**: 2026-03-14 diff --git a/.specify/scripts/bash/check-prerequisites.sh b/.specify/scripts/bash/check-prerequisites.sh new file mode 100755 index 00000000..6f7c99e0 --- /dev/null +++ b/.specify/scripts/bash/check-prerequisites.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env bash + +# Consolidated prerequisite checking script +# +# This script provides unified prerequisite checking for Spec-Driven Development workflow. +# It replaces the functionality previously spread across multiple scripts. +# +# Usage: ./check-prerequisites.sh [OPTIONS] +# +# OPTIONS: +# --json Output in JSON format +# --require-tasks Require tasks.md to exist (for implementation phase) +# --include-tasks Include tasks.md in AVAILABLE_DOCS list +# --paths-only Only output path variables (no validation) +# --help, -h Show help message +# +# OUTPUTS: +# JSON mode: {"FEATURE_DIR":"...", "AVAILABLE_DOCS":["..."]} +# Text mode: FEATURE_DIR:... \n AVAILABLE_DOCS: \n ✓/✗ file.md +# Paths only: REPO_ROOT: ... \n BRANCH: ... \n FEATURE_DIR: ... etc. + +set -e + +# Parse command line arguments +JSON_MODE=false +REQUIRE_TASKS=false +INCLUDE_TASKS=false +PATHS_ONLY=false + +for arg in "$@"; do + case "$arg" in + --json) + JSON_MODE=true + ;; + --require-tasks) + REQUIRE_TASKS=true + ;; + --include-tasks) + INCLUDE_TASKS=true + ;; + --paths-only) + PATHS_ONLY=true + ;; + --help|-h) + cat << 'EOF' +Usage: check-prerequisites.sh [OPTIONS] + +Consolidated prerequisite checking for Spec-Driven Development workflow. + +OPTIONS: + --json Output in JSON format + --require-tasks Require tasks.md to exist (for implementation phase) + --include-tasks Include tasks.md in AVAILABLE_DOCS list + --paths-only Only output path variables (no prerequisite validation) + --help, -h Show this help message + +EXAMPLES: + # Check task prerequisites (plan.md required) + ./check-prerequisites.sh --json + + # Check implementation prerequisites (plan.md + tasks.md required) + ./check-prerequisites.sh --json --require-tasks --include-tasks + + # Get feature paths only (no validation) + ./check-prerequisites.sh --paths-only + +EOF + exit 0 + ;; + *) + echo "ERROR: Unknown option '$arg'. Use --help for usage information." >&2 + exit 1 + ;; + esac +done + +# Source common functions +SCRIPT_DIR="$(CDPATH="" cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" + +# Get feature paths and validate branch +_paths_output=$(get_feature_paths) || { echo "ERROR: Failed to resolve feature paths" >&2; exit 1; } +eval "$_paths_output" +unset _paths_output +check_feature_branch "$CURRENT_BRANCH" "$HAS_GIT" || exit 1 + +# If paths-only mode, output paths and exit (support JSON + paths-only combined) +if $PATHS_ONLY; then + if $JSON_MODE; then + # Minimal JSON paths payload (no validation performed) + if has_jq; then + jq -cn \ + --arg repo_root "$REPO_ROOT" \ + --arg branch "$CURRENT_BRANCH" \ + --arg feature_dir "$FEATURE_DIR" \ + --arg feature_spec "$FEATURE_SPEC" \ + --arg impl_plan "$IMPL_PLAN" \ + --arg tasks "$TASKS" \ + '{REPO_ROOT:$repo_root,BRANCH:$branch,FEATURE_DIR:$feature_dir,FEATURE_SPEC:$feature_spec,IMPL_PLAN:$impl_plan,TASKS:$tasks}' + else + printf '{"REPO_ROOT":"%s","BRANCH":"%s","FEATURE_DIR":"%s","FEATURE_SPEC":"%s","IMPL_PLAN":"%s","TASKS":"%s"}\n' \ + "$(json_escape "$REPO_ROOT")" "$(json_escape "$CURRENT_BRANCH")" "$(json_escape "$FEATURE_DIR")" "$(json_escape "$FEATURE_SPEC")" "$(json_escape "$IMPL_PLAN")" "$(json_escape "$TASKS")" + fi + else + echo "REPO_ROOT: $REPO_ROOT" + echo "BRANCH: $CURRENT_BRANCH" + echo "FEATURE_DIR: $FEATURE_DIR" + echo "FEATURE_SPEC: $FEATURE_SPEC" + echo "IMPL_PLAN: $IMPL_PLAN" + echo "TASKS: $TASKS" + fi + exit 0 +fi + +# Validate required directories and files +if [[ ! -d "$FEATURE_DIR" ]]; then + echo "ERROR: Feature directory not found: $FEATURE_DIR" >&2 + echo "Run /speckit.specify first to create the feature structure." >&2 + exit 1 +fi + +if [[ ! -f "$IMPL_PLAN" ]]; then + echo "ERROR: plan.md not found in $FEATURE_DIR" >&2 + echo "Run /speckit.plan first to create the implementation plan." >&2 + exit 1 +fi + +# Check for tasks.md if required +if $REQUIRE_TASKS && [[ ! -f "$TASKS" ]]; then + echo "ERROR: tasks.md not found in $FEATURE_DIR" >&2 + echo "Run /speckit.tasks first to create the task list." >&2 + exit 1 +fi + +# Build list of available documents +docs=() + +# Always check these optional docs +[[ -f "$RESEARCH" ]] && docs+=("research.md") +[[ -f "$DATA_MODEL" ]] && docs+=("data-model.md") + +# Check contracts directory (only if it exists and has files) +if [[ -d "$CONTRACTS_DIR" ]] && [[ -n "$(ls -A "$CONTRACTS_DIR" 2>/dev/null)" ]]; then + docs+=("contracts/") +fi + +[[ -f "$QUICKSTART" ]] && docs+=("quickstart.md") + +# Include tasks.md if requested and it exists +if $INCLUDE_TASKS && [[ -f "$TASKS" ]]; then + docs+=("tasks.md") +fi + +# Output results +if $JSON_MODE; then + # Build JSON array of documents + if has_jq; then + if [[ ${#docs[@]} -eq 0 ]]; then + json_docs="[]" + else + json_docs=$(printf '%s\n' "${docs[@]}" | jq -R . | jq -s .) + fi + jq -cn \ + --arg feature_dir "$FEATURE_DIR" \ + --argjson docs "$json_docs" \ + '{FEATURE_DIR:$feature_dir,AVAILABLE_DOCS:$docs}' + else + if [[ ${#docs[@]} -eq 0 ]]; then + json_docs="[]" + else + json_docs=$(printf '"%s",' "${docs[@]}") + json_docs="[${json_docs%,}]" + fi + printf '{"FEATURE_DIR":"%s","AVAILABLE_DOCS":%s}\n' "$(json_escape "$FEATURE_DIR")" "$json_docs" + fi +else + # Text output + echo "FEATURE_DIR:$FEATURE_DIR" + echo "AVAILABLE_DOCS:" + + # Show status of each potential document + check_file "$RESEARCH" "research.md" + check_file "$DATA_MODEL" "data-model.md" + check_dir "$CONTRACTS_DIR" "contracts/" + check_file "$QUICKSTART" "quickstart.md" + + if $INCLUDE_TASKS; then + check_file "$TASKS" "tasks.md" + fi +fi diff --git a/.specify/scripts/bash/common.sh b/.specify/scripts/bash/common.sh new file mode 100755 index 00000000..52e363e6 --- /dev/null +++ b/.specify/scripts/bash/common.sh @@ -0,0 +1,253 @@ +#!/usr/bin/env bash +# Common functions and variables for all scripts + +# Get repository root, with fallback for non-git repositories +get_repo_root() { + if git rev-parse --show-toplevel >/dev/null 2>&1; then + git rev-parse --show-toplevel + else + # Fall back to script location for non-git repos + local script_dir="$(CDPATH="" cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + (cd "$script_dir/../../.." && pwd) + fi +} + +# Get current branch, with fallback for non-git repositories +get_current_branch() { + # First check if SPECIFY_FEATURE environment variable is set + if [[ -n "${SPECIFY_FEATURE:-}" ]]; then + echo "$SPECIFY_FEATURE" + return + fi + + # Then check git if available + if git rev-parse --abbrev-ref HEAD >/dev/null 2>&1; then + git rev-parse --abbrev-ref HEAD + return + fi + + # For non-git repos, try to find the latest feature directory + local repo_root=$(get_repo_root) + local specs_dir="$repo_root/specs" + + if [[ -d "$specs_dir" ]]; then + local latest_feature="" + local highest=0 + + for dir in "$specs_dir"/*; do + if [[ -d "$dir" ]]; then + local dirname=$(basename "$dir") + if [[ "$dirname" =~ ^([0-9]{3})- ]]; then + local number=${BASH_REMATCH[1]} + number=$((10#$number)) + if [[ "$number" -gt "$highest" ]]; then + highest=$number + latest_feature=$dirname + fi + fi + fi + done + + if [[ -n "$latest_feature" ]]; then + echo "$latest_feature" + return + fi + fi + + echo "main" # Final fallback +} + +# Check if we have git available +has_git() { + git rev-parse --show-toplevel >/dev/null 2>&1 +} + +check_feature_branch() { + local branch="$1" + local has_git_repo="$2" + + # For non-git repos, we can't enforce branch naming but still provide output + if [[ "$has_git_repo" != "true" ]]; then + echo "[specify] Warning: Git repository not detected; skipped branch validation" >&2 + return 0 + fi + + if [[ ! "$branch" =~ ^[0-9]{3}- ]]; then + echo "ERROR: Not on a feature branch. Current branch: $branch" >&2 + echo "Feature branches should be named like: 001-feature-name" >&2 + return 1 + fi + + return 0 +} + +get_feature_dir() { echo "$1/specs/$2"; } + +# Find feature directory by numeric prefix instead of exact branch match +# This allows multiple branches to work on the same spec (e.g., 004-fix-bug, 004-add-feature) +find_feature_dir_by_prefix() { + local repo_root="$1" + local branch_name="$2" + local specs_dir="$repo_root/specs" + + # Extract numeric prefix from branch (e.g., "004" from "004-whatever") + if [[ ! "$branch_name" =~ ^([0-9]{3})- ]]; then + # If branch doesn't have numeric prefix, fall back to exact match + echo "$specs_dir/$branch_name" + return + fi + + local prefix="${BASH_REMATCH[1]}" + + # Search for directories in specs/ that start with this prefix + local matches=() + if [[ -d "$specs_dir" ]]; then + for dir in "$specs_dir"/"$prefix"-*; do + if [[ -d "$dir" ]]; then + matches+=("$(basename "$dir")") + fi + done + fi + + # Handle results + if [[ ${#matches[@]} -eq 0 ]]; then + # No match found - return the branch name path (will fail later with clear error) + echo "$specs_dir/$branch_name" + elif [[ ${#matches[@]} -eq 1 ]]; then + # Exactly one match - perfect! + echo "$specs_dir/${matches[0]}" + else + # Multiple matches - this shouldn't happen with proper naming convention + echo "ERROR: Multiple spec directories found with prefix '$prefix': ${matches[*]}" >&2 + echo "Please ensure only one spec directory exists per numeric prefix." >&2 + return 1 + fi +} + +get_feature_paths() { + local repo_root=$(get_repo_root) + local current_branch=$(get_current_branch) + local has_git_repo="false" + + if has_git; then + has_git_repo="true" + fi + + # Use prefix-based lookup to support multiple branches per spec + local feature_dir + if ! feature_dir=$(find_feature_dir_by_prefix "$repo_root" "$current_branch"); then + echo "ERROR: Failed to resolve feature directory" >&2 + return 1 + fi + + # Use printf '%q' to safely quote values, preventing shell injection + # via crafted branch names or paths containing special characters + printf 'REPO_ROOT=%q\n' "$repo_root" + printf 'CURRENT_BRANCH=%q\n' "$current_branch" + printf 'HAS_GIT=%q\n' "$has_git_repo" + printf 'FEATURE_DIR=%q\n' "$feature_dir" + printf 'FEATURE_SPEC=%q\n' "$feature_dir/spec.md" + printf 'IMPL_PLAN=%q\n' "$feature_dir/plan.md" + printf 'TASKS=%q\n' "$feature_dir/tasks.md" + printf 'RESEARCH=%q\n' "$feature_dir/research.md" + printf 'DATA_MODEL=%q\n' "$feature_dir/data-model.md" + printf 'QUICKSTART=%q\n' "$feature_dir/quickstart.md" + printf 'CONTRACTS_DIR=%q\n' "$feature_dir/contracts" +} + +# Check if jq is available for safe JSON construction +has_jq() { + command -v jq >/dev/null 2>&1 +} + +# Escape a string for safe embedding in a JSON value (fallback when jq is unavailable). +# Handles backslash, double-quote, and control characters (newline, tab, carriage return). +json_escape() { + local s="$1" + s="${s//\\/\\\\}" + s="${s//\"/\\\"}" + s="${s//$'\n'/\\n}" + s="${s//$'\t'/\\t}" + s="${s//$'\r'/\\r}" + printf '%s' "$s" +} + +check_file() { [[ -f "$1" ]] && echo " ✓ $2" || echo " ✗ $2"; } +check_dir() { [[ -d "$1" && -n $(ls -A "$1" 2>/dev/null) ]] && echo " ✓ $2" || echo " ✗ $2"; } + +# Resolve a template name to a file path using the priority stack: +# 1. .specify/templates/overrides/ +# 2. .specify/presets//templates/ (sorted by priority from .registry) +# 3. .specify/extensions//templates/ +# 4. .specify/templates/ (core) +resolve_template() { + local template_name="$1" + local repo_root="$2" + local base="$repo_root/.specify/templates" + + # Priority 1: Project overrides + local override="$base/overrides/${template_name}.md" + [ -f "$override" ] && echo "$override" && return 0 + + # Priority 2: Installed presets (sorted by priority from .registry) + local presets_dir="$repo_root/.specify/presets" + if [ -d "$presets_dir" ]; then + local registry_file="$presets_dir/.registry" + if [ -f "$registry_file" ] && command -v python3 >/dev/null 2>&1; then + # Read preset IDs sorted by priority (lower number = higher precedence) + local sorted_presets + sorted_presets=$(SPECKIT_REGISTRY="$registry_file" python3 -c " +import json, sys, os +try: + with open(os.environ['SPECKIT_REGISTRY']) as f: + data = json.load(f) + presets = data.get('presets', {}) + for pid, meta in sorted(presets.items(), key=lambda x: x[1].get('priority', 10)): + print(pid) +except Exception: + sys.exit(1) +" 2>/dev/null) + if [ $? -eq 0 ] && [ -n "$sorted_presets" ]; then + while IFS= read -r preset_id; do + local candidate="$presets_dir/$preset_id/templates/${template_name}.md" + [ -f "$candidate" ] && echo "$candidate" && return 0 + done <<< "$sorted_presets" + else + # python3 returned empty list — fall through to directory scan + for preset in "$presets_dir"/*/; do + [ -d "$preset" ] || continue + local candidate="$preset/templates/${template_name}.md" + [ -f "$candidate" ] && echo "$candidate" && return 0 + done + fi + else + # Fallback: alphabetical directory order (no python3 available) + for preset in "$presets_dir"/*/; do + [ -d "$preset" ] || continue + local candidate="$preset/templates/${template_name}.md" + [ -f "$candidate" ] && echo "$candidate" && return 0 + done + fi + fi + + # Priority 3: Extension-provided templates + local ext_dir="$repo_root/.specify/extensions" + if [ -d "$ext_dir" ]; then + for ext in "$ext_dir"/*/; do + [ -d "$ext" ] || continue + # Skip hidden directories (e.g. .backup, .cache) + case "$(basename "$ext")" in .*) continue;; esac + local candidate="$ext/templates/${template_name}.md" + [ -f "$candidate" ] && echo "$candidate" && return 0 + done + fi + + # Priority 4: Core templates + local core="$base/${template_name}.md" + [ -f "$core" ] && echo "$core" && return 0 + + # Return success with empty output so callers using set -e don't abort; + # callers check [ -n "$TEMPLATE" ] to detect "not found". + return 0 +} + diff --git a/.specify/scripts/bash/create-new-feature.sh b/.specify/scripts/bash/create-new-feature.sh new file mode 100755 index 00000000..0823cca2 --- /dev/null +++ b/.specify/scripts/bash/create-new-feature.sh @@ -0,0 +1,333 @@ +#!/usr/bin/env bash + +set -e + +JSON_MODE=false +SHORT_NAME="" +BRANCH_NUMBER="" +ARGS=() +i=1 +while [ $i -le $# ]; do + arg="${!i}" + case "$arg" in + --json) + JSON_MODE=true + ;; + --short-name) + if [ $((i + 1)) -gt $# ]; then + echo 'Error: --short-name requires a value' >&2 + exit 1 + fi + i=$((i + 1)) + next_arg="${!i}" + # Check if the next argument is another option (starts with --) + if [[ "$next_arg" == --* ]]; then + echo 'Error: --short-name requires a value' >&2 + exit 1 + fi + SHORT_NAME="$next_arg" + ;; + --number) + if [ $((i + 1)) -gt $# ]; then + echo 'Error: --number requires a value' >&2 + exit 1 + fi + i=$((i + 1)) + next_arg="${!i}" + if [[ "$next_arg" == --* ]]; then + echo 'Error: --number requires a value' >&2 + exit 1 + fi + BRANCH_NUMBER="$next_arg" + ;; + --help|-h) + echo "Usage: $0 [--json] [--short-name ] [--number N] " + echo "" + echo "Options:" + echo " --json Output in JSON format" + echo " --short-name Provide a custom short name (2-4 words) for the branch" + echo " --number N Specify branch number manually (overrides auto-detection)" + echo " --help, -h Show this help message" + echo "" + echo "Examples:" + echo " $0 'Add user authentication system' --short-name 'user-auth'" + echo " $0 'Implement OAuth2 integration for API' --number 5" + exit 0 + ;; + *) + ARGS+=("$arg") + ;; + esac + i=$((i + 1)) +done + +FEATURE_DESCRIPTION="${ARGS[*]}" +if [ -z "$FEATURE_DESCRIPTION" ]; then + echo "Usage: $0 [--json] [--short-name ] [--number N] " >&2 + exit 1 +fi + +# Trim whitespace and validate description is not empty (e.g., user passed only whitespace) +FEATURE_DESCRIPTION=$(echo "$FEATURE_DESCRIPTION" | xargs) +if [ -z "$FEATURE_DESCRIPTION" ]; then + echo "Error: Feature description cannot be empty or contain only whitespace" >&2 + exit 1 +fi + +# Function to find the repository root by searching for existing project markers +find_repo_root() { + local dir="$1" + while [ "$dir" != "/" ]; do + if [ -d "$dir/.git" ] || [ -d "$dir/.specify" ]; then + echo "$dir" + return 0 + fi + dir="$(dirname "$dir")" + done + return 1 +} + +# Function to get highest number from specs directory +get_highest_from_specs() { + local specs_dir="$1" + local highest=0 + + if [ -d "$specs_dir" ]; then + for dir in "$specs_dir"/*; do + [ -d "$dir" ] || continue + dirname=$(basename "$dir") + number=$(echo "$dirname" | grep -o '^[0-9]\+' || echo "0") + number=$((10#$number)) + if [ "$number" -gt "$highest" ]; then + highest=$number + fi + done + fi + + echo "$highest" +} + +# Function to get highest number from git branches +get_highest_from_branches() { + local highest=0 + + # Get all branches (local and remote) + branches=$(git branch -a 2>/dev/null || echo "") + + if [ -n "$branches" ]; then + while IFS= read -r branch; do + # Clean branch name: remove leading markers and remote prefixes + clean_branch=$(echo "$branch" | sed 's/^[* ]*//; s|^remotes/[^/]*/||') + + # Extract feature number if branch matches pattern ###-* + if echo "$clean_branch" | grep -q '^[0-9]\{3\}-'; then + number=$(echo "$clean_branch" | grep -o '^[0-9]\{3\}' || echo "0") + number=$((10#$number)) + if [ "$number" -gt "$highest" ]; then + highest=$number + fi + fi + done <<< "$branches" + fi + + echo "$highest" +} + +# Function to check existing branches (local and remote) and return next available number +check_existing_branches() { + local specs_dir="$1" + + # Fetch all remotes to get latest branch info (suppress errors if no remotes) + git fetch --all --prune 2>/dev/null || true + + # Get highest number from ALL branches (not just matching short name) + local highest_branch=$(get_highest_from_branches) + + # Get highest number from ALL specs (not just matching short name) + local highest_spec=$(get_highest_from_specs "$specs_dir") + + # Take the maximum of both + local max_num=$highest_branch + if [ "$highest_spec" -gt "$max_num" ]; then + max_num=$highest_spec + fi + + # Return next number + echo $((max_num + 1)) +} + +# Function to clean and format a branch name +clean_branch_name() { + local name="$1" + echo "$name" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/-/g' | sed 's/-\+/-/g' | sed 's/^-//' | sed 's/-$//' +} + +# Escape a string for safe embedding in a JSON value (fallback when jq is unavailable). +json_escape() { + local s="$1" + s="${s//\\/\\\\}" + s="${s//\"/\\\"}" + s="${s//$'\n'/\\n}" + s="${s//$'\t'/\\t}" + s="${s//$'\r'/\\r}" + printf '%s' "$s" +} + +# Resolve repository root. Prefer git information when available, but fall back +# to searching for repository markers so the workflow still functions in repositories that +# were initialised with --no-git. +SCRIPT_DIR="$(CDPATH="" cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" + +if git rev-parse --show-toplevel >/dev/null 2>&1; then + REPO_ROOT=$(git rev-parse --show-toplevel) + HAS_GIT=true +else + REPO_ROOT="$(find_repo_root "$SCRIPT_DIR")" + if [ -z "$REPO_ROOT" ]; then + echo "Error: Could not determine repository root. Please run this script from within the repository." >&2 + exit 1 + fi + HAS_GIT=false +fi + +cd "$REPO_ROOT" + +SPECS_DIR="$REPO_ROOT/specs" +mkdir -p "$SPECS_DIR" + +# Function to generate branch name with stop word filtering and length filtering +generate_branch_name() { + local description="$1" + + # Common stop words to filter out + local stop_words="^(i|a|an|the|to|for|of|in|on|at|by|with|from|is|are|was|were|be|been|being|have|has|had|do|does|did|will|would|should|could|can|may|might|must|shall|this|that|these|those|my|your|our|their|want|need|add|get|set)$" + + # Convert to lowercase and split into words + local clean_name=$(echo "$description" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/ /g') + + # Filter words: remove stop words and words shorter than 3 chars (unless they're uppercase acronyms in original) + local meaningful_words=() + for word in $clean_name; do + # Skip empty words + [ -z "$word" ] && continue + + # Keep words that are NOT stop words AND (length >= 3 OR are potential acronyms) + if ! echo "$word" | grep -qiE "$stop_words"; then + if [ ${#word} -ge 3 ]; then + meaningful_words+=("$word") + elif echo "$description" | grep -q "\b${word^^}\b"; then + # Keep short words if they appear as uppercase in original (likely acronyms) + meaningful_words+=("$word") + fi + fi + done + + # If we have meaningful words, use first 3-4 of them + if [ ${#meaningful_words[@]} -gt 0 ]; then + local max_words=3 + if [ ${#meaningful_words[@]} -eq 4 ]; then max_words=4; fi + + local result="" + local count=0 + for word in "${meaningful_words[@]}"; do + if [ $count -ge $max_words ]; then break; fi + if [ -n "$result" ]; then result="$result-"; fi + result="$result$word" + count=$((count + 1)) + done + echo "$result" + else + # Fallback to original logic if no meaningful words found + local cleaned=$(clean_branch_name "$description") + echo "$cleaned" | tr '-' '\n' | grep -v '^$' | head -3 | tr '\n' '-' | sed 's/-$//' + fi +} + +# Generate branch name +if [ -n "$SHORT_NAME" ]; then + # Use provided short name, just clean it up + BRANCH_SUFFIX=$(clean_branch_name "$SHORT_NAME") +else + # Generate from description with smart filtering + BRANCH_SUFFIX=$(generate_branch_name "$FEATURE_DESCRIPTION") +fi + +# Determine branch number +if [ -z "$BRANCH_NUMBER" ]; then + if [ "$HAS_GIT" = true ]; then + # Check existing branches on remotes + BRANCH_NUMBER=$(check_existing_branches "$SPECS_DIR") + else + # Fall back to local directory check + HIGHEST=$(get_highest_from_specs "$SPECS_DIR") + BRANCH_NUMBER=$((HIGHEST + 1)) + fi +fi + +# Force base-10 interpretation to prevent octal conversion (e.g., 010 → 8 in octal, but should be 10 in decimal) +FEATURE_NUM=$(printf "%03d" "$((10#$BRANCH_NUMBER))") +BRANCH_NAME="${FEATURE_NUM}-${BRANCH_SUFFIX}" + +# GitHub enforces a 244-byte limit on branch names +# Validate and truncate if necessary +MAX_BRANCH_LENGTH=244 +if [ ${#BRANCH_NAME} -gt $MAX_BRANCH_LENGTH ]; then + # Calculate how much we need to trim from suffix + # Account for: feature number (3) + hyphen (1) = 4 chars + MAX_SUFFIX_LENGTH=$((MAX_BRANCH_LENGTH - 4)) + + # Truncate suffix at word boundary if possible + TRUNCATED_SUFFIX=$(echo "$BRANCH_SUFFIX" | cut -c1-$MAX_SUFFIX_LENGTH) + # Remove trailing hyphen if truncation created one + TRUNCATED_SUFFIX=$(echo "$TRUNCATED_SUFFIX" | sed 's/-$//') + + ORIGINAL_BRANCH_NAME="$BRANCH_NAME" + BRANCH_NAME="${FEATURE_NUM}-${TRUNCATED_SUFFIX}" + + >&2 echo "[specify] Warning: Branch name exceeded GitHub's 244-byte limit" + >&2 echo "[specify] Original: $ORIGINAL_BRANCH_NAME (${#ORIGINAL_BRANCH_NAME} bytes)" + >&2 echo "[specify] Truncated to: $BRANCH_NAME (${#BRANCH_NAME} bytes)" +fi + +if [ "$HAS_GIT" = true ]; then + if ! git checkout -b "$BRANCH_NAME" 2>/dev/null; then + # Check if branch already exists + if git branch --list "$BRANCH_NAME" | grep -q .; then + >&2 echo "Error: Branch '$BRANCH_NAME' already exists. Please use a different feature name or specify a different number with --number." + exit 1 + else + >&2 echo "Error: Failed to create git branch '$BRANCH_NAME'. Please check your git configuration and try again." + exit 1 + fi + fi +else + >&2 echo "[specify] Warning: Git repository not detected; skipped branch creation for $BRANCH_NAME" +fi + +FEATURE_DIR="$SPECS_DIR/$BRANCH_NAME" +mkdir -p "$FEATURE_DIR" + +TEMPLATE=$(resolve_template "spec-template" "$REPO_ROOT") +SPEC_FILE="$FEATURE_DIR/spec.md" +if [ -n "$TEMPLATE" ] && [ -f "$TEMPLATE" ]; then cp "$TEMPLATE" "$SPEC_FILE"; else touch "$SPEC_FILE"; fi + +# Inform the user how to persist the feature variable in their own shell +printf '# To persist: export SPECIFY_FEATURE=%q\n' "$BRANCH_NAME" >&2 + +if $JSON_MODE; then + if command -v jq >/dev/null 2>&1; then + jq -cn \ + --arg branch_name "$BRANCH_NAME" \ + --arg spec_file "$SPEC_FILE" \ + --arg feature_num "$FEATURE_NUM" \ + '{BRANCH_NAME:$branch_name,SPEC_FILE:$spec_file,FEATURE_NUM:$feature_num}' + else + printf '{"BRANCH_NAME":"%s","SPEC_FILE":"%s","FEATURE_NUM":"%s"}\n' "$(json_escape "$BRANCH_NAME")" "$(json_escape "$SPEC_FILE")" "$(json_escape "$FEATURE_NUM")" + fi +else + echo "BRANCH_NAME: $BRANCH_NAME" + echo "SPEC_FILE: $SPEC_FILE" + echo "FEATURE_NUM: $FEATURE_NUM" + printf '# To persist in your shell: export SPECIFY_FEATURE=%q\n' "$BRANCH_NAME" +fi diff --git a/.specify/scripts/bash/setup-plan.sh b/.specify/scripts/bash/setup-plan.sh new file mode 100755 index 00000000..2a044c67 --- /dev/null +++ b/.specify/scripts/bash/setup-plan.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash + +set -e + +# Parse command line arguments +JSON_MODE=false +ARGS=() + +for arg in "$@"; do + case "$arg" in + --json) + JSON_MODE=true + ;; + --help|-h) + echo "Usage: $0 [--json]" + echo " --json Output results in JSON format" + echo " --help Show this help message" + exit 0 + ;; + *) + ARGS+=("$arg") + ;; + esac +done + +# Get script directory and load common functions +SCRIPT_DIR="$(CDPATH="" cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" + +# Get all paths and variables from common functions +_paths_output=$(get_feature_paths) || { echo "ERROR: Failed to resolve feature paths" >&2; exit 1; } +eval "$_paths_output" +unset _paths_output + +# Check if we're on a proper feature branch (only for git repos) +check_feature_branch "$CURRENT_BRANCH" "$HAS_GIT" || exit 1 + +# Ensure the feature directory exists +mkdir -p "$FEATURE_DIR" + +# Copy plan template if it exists +TEMPLATE=$(resolve_template "plan-template" "$REPO_ROOT") +if [[ -n "$TEMPLATE" ]] && [[ -f "$TEMPLATE" ]]; then + cp "$TEMPLATE" "$IMPL_PLAN" + echo "Copied plan template to $IMPL_PLAN" +else + echo "Warning: Plan template not found" + # Create a basic plan file if template doesn't exist + touch "$IMPL_PLAN" +fi + +# Output results +if $JSON_MODE; then + if has_jq; then + jq -cn \ + --arg feature_spec "$FEATURE_SPEC" \ + --arg impl_plan "$IMPL_PLAN" \ + --arg specs_dir "$FEATURE_DIR" \ + --arg branch "$CURRENT_BRANCH" \ + --arg has_git "$HAS_GIT" \ + '{FEATURE_SPEC:$feature_spec,IMPL_PLAN:$impl_plan,SPECS_DIR:$specs_dir,BRANCH:$branch,HAS_GIT:$has_git}' + else + printf '{"FEATURE_SPEC":"%s","IMPL_PLAN":"%s","SPECS_DIR":"%s","BRANCH":"%s","HAS_GIT":"%s"}\n' \ + "$(json_escape "$FEATURE_SPEC")" "$(json_escape "$IMPL_PLAN")" "$(json_escape "$FEATURE_DIR")" "$(json_escape "$CURRENT_BRANCH")" "$(json_escape "$HAS_GIT")" + fi +else + echo "FEATURE_SPEC: $FEATURE_SPEC" + echo "IMPL_PLAN: $IMPL_PLAN" + echo "SPECS_DIR: $FEATURE_DIR" + echo "BRANCH: $CURRENT_BRANCH" + echo "HAS_GIT: $HAS_GIT" +fi + diff --git a/.specify/scripts/bash/update-agent-context.sh b/.specify/scripts/bash/update-agent-context.sh new file mode 100755 index 00000000..e0f28548 --- /dev/null +++ b/.specify/scripts/bash/update-agent-context.sh @@ -0,0 +1,808 @@ +#!/usr/bin/env bash + +# Update agent context files with information from plan.md +# +# This script maintains AI agent context files by parsing feature specifications +# and updating agent-specific configuration files with project information. +# +# MAIN FUNCTIONS: +# 1. Environment Validation +# - Verifies git repository structure and branch information +# - Checks for required plan.md files and templates +# - Validates file permissions and accessibility +# +# 2. Plan Data Extraction +# - Parses plan.md files to extract project metadata +# - Identifies language/version, frameworks, databases, and project types +# - Handles missing or incomplete specification data gracefully +# +# 3. Agent File Management +# - Creates new agent context files from templates when needed +# - Updates existing agent files with new project information +# - Preserves manual additions and custom configurations +# - Supports multiple AI agent formats and directory structures +# +# 4. Content Generation +# - Generates language-specific build/test commands +# - Creates appropriate project directory structures +# - Updates technology stacks and recent changes sections +# - Maintains consistent formatting and timestamps +# +# 5. Multi-Agent Support +# - Handles agent-specific file paths and naming conventions +# - Supports: Claude, Gemini, Copilot, Cursor, Qwen, opencode, Codex, Windsurf, Kilo Code, Auggie CLI, Roo Code, CodeBuddy CLI, Qoder CLI, Amp, SHAI, Tabnine CLI, Kiro CLI, Mistral Vibe, Kimi Code, Antigravity or Generic +# - Can update single agents or all existing agent files +# - Creates default Claude file if no agent files exist +# +# Usage: ./update-agent-context.sh [agent_type] +# Agent types: claude|gemini|copilot|cursor-agent|qwen|opencode|codex|windsurf|kilocode|auggie|roo|codebuddy|amp|shai|tabnine|kiro-cli|agy|bob|vibe|qodercli|kimi|generic +# Leave empty to update all existing agent files + +set -e + +# Enable strict error handling +set -u +set -o pipefail + +#============================================================================== +# Configuration and Global Variables +#============================================================================== + +# Get script directory and load common functions +SCRIPT_DIR="$(CDPATH="" cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/common.sh" + +# Get all paths and variables from common functions +_paths_output=$(get_feature_paths) || { echo "ERROR: Failed to resolve feature paths" >&2; exit 1; } +eval "$_paths_output" +unset _paths_output + +NEW_PLAN="$IMPL_PLAN" # Alias for compatibility with existing code +AGENT_TYPE="${1:-}" + +# Agent-specific file paths +CLAUDE_FILE="$REPO_ROOT/CLAUDE.md" +GEMINI_FILE="$REPO_ROOT/GEMINI.md" +COPILOT_FILE="$REPO_ROOT/.github/agents/copilot-instructions.md" +CURSOR_FILE="$REPO_ROOT/.cursor/rules/specify-rules.mdc" +QWEN_FILE="$REPO_ROOT/QWEN.md" +AGENTS_FILE="$REPO_ROOT/AGENTS.md" +WINDSURF_FILE="$REPO_ROOT/.windsurf/rules/specify-rules.md" +KILOCODE_FILE="$REPO_ROOT/.kilocode/rules/specify-rules.md" +AUGGIE_FILE="$REPO_ROOT/.augment/rules/specify-rules.md" +ROO_FILE="$REPO_ROOT/.roo/rules/specify-rules.md" +CODEBUDDY_FILE="$REPO_ROOT/CODEBUDDY.md" +QODER_FILE="$REPO_ROOT/QODER.md" +# AMP, Kiro CLI, and IBM Bob all share AGENTS.md — use AGENTS_FILE to avoid +# updating the same file multiple times. +AMP_FILE="$AGENTS_FILE" +SHAI_FILE="$REPO_ROOT/SHAI.md" +TABNINE_FILE="$REPO_ROOT/TABNINE.md" +KIRO_FILE="$AGENTS_FILE" +AGY_FILE="$REPO_ROOT/.agent/rules/specify-rules.md" +BOB_FILE="$AGENTS_FILE" +VIBE_FILE="$REPO_ROOT/.vibe/agents/specify-agents.md" +KIMI_FILE="$REPO_ROOT/KIMI.md" + +# Template file +TEMPLATE_FILE="$REPO_ROOT/.specify/templates/agent-file-template.md" + +# Global variables for parsed plan data +NEW_LANG="" +NEW_FRAMEWORK="" +NEW_DB="" +NEW_PROJECT_TYPE="" + +#============================================================================== +# Utility Functions +#============================================================================== + +log_info() { + echo "INFO: $1" +} + +log_success() { + echo "✓ $1" +} + +log_error() { + echo "ERROR: $1" >&2 +} + +log_warning() { + echo "WARNING: $1" >&2 +} + +# Cleanup function for temporary files +cleanup() { + local exit_code=$? + # Disarm traps to prevent re-entrant loop + trap - EXIT INT TERM + rm -f /tmp/agent_update_*_$$ + rm -f /tmp/manual_additions_$$ + exit $exit_code +} + +# Set up cleanup trap +trap cleanup EXIT INT TERM + +#============================================================================== +# Validation Functions +#============================================================================== + +validate_environment() { + # Check if we have a current branch/feature (git or non-git) + if [[ -z "$CURRENT_BRANCH" ]]; then + log_error "Unable to determine current feature" + if [[ "$HAS_GIT" == "true" ]]; then + log_info "Make sure you're on a feature branch" + else + log_info "Set SPECIFY_FEATURE environment variable or create a feature first" + fi + exit 1 + fi + + # Check if plan.md exists + if [[ ! -f "$NEW_PLAN" ]]; then + log_error "No plan.md found at $NEW_PLAN" + log_info "Make sure you're working on a feature with a corresponding spec directory" + if [[ "$HAS_GIT" != "true" ]]; then + log_info "Use: export SPECIFY_FEATURE=your-feature-name or create a new feature first" + fi + exit 1 + fi + + # Check if template exists (needed for new files) + if [[ ! -f "$TEMPLATE_FILE" ]]; then + log_warning "Template file not found at $TEMPLATE_FILE" + log_warning "Creating new agent files will fail" + fi +} + +#============================================================================== +# Plan Parsing Functions +#============================================================================== + +extract_plan_field() { + local field_pattern="$1" + local plan_file="$2" + + grep "^\*\*${field_pattern}\*\*: " "$plan_file" 2>/dev/null | \ + head -1 | \ + sed "s|^\*\*${field_pattern}\*\*: ||" | \ + sed 's/^[ \t]*//;s/[ \t]*$//' | \ + grep -v "NEEDS CLARIFICATION" | \ + grep -v "^N/A$" || echo "" +} + +parse_plan_data() { + local plan_file="$1" + + if [[ ! -f "$plan_file" ]]; then + log_error "Plan file not found: $plan_file" + return 1 + fi + + if [[ ! -r "$plan_file" ]]; then + log_error "Plan file is not readable: $plan_file" + return 1 + fi + + log_info "Parsing plan data from $plan_file" + + NEW_LANG=$(extract_plan_field "Language/Version" "$plan_file") + NEW_FRAMEWORK=$(extract_plan_field "Primary Dependencies" "$plan_file") + NEW_DB=$(extract_plan_field "Storage" "$plan_file") + NEW_PROJECT_TYPE=$(extract_plan_field "Project Type" "$plan_file") + + # Log what we found + if [[ -n "$NEW_LANG" ]]; then + log_info "Found language: $NEW_LANG" + else + log_warning "No language information found in plan" + fi + + if [[ -n "$NEW_FRAMEWORK" ]]; then + log_info "Found framework: $NEW_FRAMEWORK" + fi + + if [[ -n "$NEW_DB" ]] && [[ "$NEW_DB" != "N/A" ]]; then + log_info "Found database: $NEW_DB" + fi + + if [[ -n "$NEW_PROJECT_TYPE" ]]; then + log_info "Found project type: $NEW_PROJECT_TYPE" + fi +} + +format_technology_stack() { + local lang="$1" + local framework="$2" + local parts=() + + # Add non-empty parts + [[ -n "$lang" && "$lang" != "NEEDS CLARIFICATION" ]] && parts+=("$lang") + [[ -n "$framework" && "$framework" != "NEEDS CLARIFICATION" && "$framework" != "N/A" ]] && parts+=("$framework") + + # Join with proper formatting + if [[ ${#parts[@]} -eq 0 ]]; then + echo "" + elif [[ ${#parts[@]} -eq 1 ]]; then + echo "${parts[0]}" + else + # Join multiple parts with " + " + local result="${parts[0]}" + for ((i=1; i<${#parts[@]}; i++)); do + result="$result + ${parts[i]}" + done + echo "$result" + fi +} + +#============================================================================== +# Template and Content Generation Functions +#============================================================================== + +get_project_structure() { + local project_type="$1" + + if [[ "$project_type" == *"web"* ]]; then + echo "backend/\\nfrontend/\\ntests/" + else + echo "src/\\ntests/" + fi +} + +get_commands_for_language() { + local lang="$1" + + case "$lang" in + *"Python"*) + echo "cd src && pytest && ruff check ." + ;; + *"Rust"*) + echo "cargo test && cargo clippy" + ;; + *"JavaScript"*|*"TypeScript"*) + echo "npm test \\&\\& npm run lint" + ;; + *) + echo "# Add commands for $lang" + ;; + esac +} + +get_language_conventions() { + local lang="$1" + echo "$lang: Follow standard conventions" +} + +create_new_agent_file() { + local target_file="$1" + local temp_file="$2" + local project_name="$3" + local current_date="$4" + + if [[ ! -f "$TEMPLATE_FILE" ]]; then + log_error "Template not found at $TEMPLATE_FILE" + return 1 + fi + + if [[ ! -r "$TEMPLATE_FILE" ]]; then + log_error "Template file is not readable: $TEMPLATE_FILE" + return 1 + fi + + log_info "Creating new agent context file from template..." + + if ! cp "$TEMPLATE_FILE" "$temp_file"; then + log_error "Failed to copy template file" + return 1 + fi + + # Replace template placeholders + local project_structure + project_structure=$(get_project_structure "$NEW_PROJECT_TYPE") + + local commands + commands=$(get_commands_for_language "$NEW_LANG") + + local language_conventions + language_conventions=$(get_language_conventions "$NEW_LANG") + + # Perform substitutions with error checking using safer approach + # Escape special characters for sed by using a different delimiter or escaping + local escaped_lang=$(printf '%s\n' "$NEW_LANG" | sed 's/[\[\.*^$()+{}|]/\\&/g') + local escaped_framework=$(printf '%s\n' "$NEW_FRAMEWORK" | sed 's/[\[\.*^$()+{}|]/\\&/g') + local escaped_branch=$(printf '%s\n' "$CURRENT_BRANCH" | sed 's/[\[\.*^$()+{}|]/\\&/g') + + # Build technology stack and recent change strings conditionally + local tech_stack + if [[ -n "$escaped_lang" && -n "$escaped_framework" ]]; then + tech_stack="- $escaped_lang + $escaped_framework ($escaped_branch)" + elif [[ -n "$escaped_lang" ]]; then + tech_stack="- $escaped_lang ($escaped_branch)" + elif [[ -n "$escaped_framework" ]]; then + tech_stack="- $escaped_framework ($escaped_branch)" + else + tech_stack="- ($escaped_branch)" + fi + + local recent_change + if [[ -n "$escaped_lang" && -n "$escaped_framework" ]]; then + recent_change="- $escaped_branch: Added $escaped_lang + $escaped_framework" + elif [[ -n "$escaped_lang" ]]; then + recent_change="- $escaped_branch: Added $escaped_lang" + elif [[ -n "$escaped_framework" ]]; then + recent_change="- $escaped_branch: Added $escaped_framework" + else + recent_change="- $escaped_branch: Added" + fi + + local substitutions=( + "s|\[PROJECT NAME\]|$project_name|" + "s|\[DATE\]|$current_date|" + "s|\[EXTRACTED FROM ALL PLAN.MD FILES\]|$tech_stack|" + "s|\[ACTUAL STRUCTURE FROM PLANS\]|$project_structure|g" + "s|\[ONLY COMMANDS FOR ACTIVE TECHNOLOGIES\]|$commands|" + "s|\[LANGUAGE-SPECIFIC, ONLY FOR LANGUAGES IN USE\]|$language_conventions|" + "s|\[LAST 3 FEATURES AND WHAT THEY ADDED\]|$recent_change|" + ) + + for substitution in "${substitutions[@]}"; do + if ! sed -i.bak -e "$substitution" "$temp_file"; then + log_error "Failed to perform substitution: $substitution" + rm -f "$temp_file" "$temp_file.bak" + return 1 + fi + done + + # Convert \n sequences to actual newlines + newline=$(printf '\n') + sed -i.bak2 "s/\\\\n/${newline}/g" "$temp_file" + + # Clean up backup files + rm -f "$temp_file.bak" "$temp_file.bak2" + + # Prepend Cursor frontmatter for .mdc files so rules are auto-included + if [[ "$target_file" == *.mdc ]]; then + local frontmatter_file + frontmatter_file=$(mktemp) || return 1 + printf '%s\n' "---" "description: Project Development Guidelines" "globs: [\"**/*\"]" "alwaysApply: true" "---" "" > "$frontmatter_file" + cat "$temp_file" >> "$frontmatter_file" + mv "$frontmatter_file" "$temp_file" + fi + + return 0 +} + + + + +update_existing_agent_file() { + local target_file="$1" + local current_date="$2" + + log_info "Updating existing agent context file..." + + # Use a single temporary file for atomic update + local temp_file + temp_file=$(mktemp) || { + log_error "Failed to create temporary file" + return 1 + } + + # Process the file in one pass + local tech_stack=$(format_technology_stack "$NEW_LANG" "$NEW_FRAMEWORK") + local new_tech_entries=() + local new_change_entry="" + + # Prepare new technology entries + if [[ -n "$tech_stack" ]] && ! grep -q "$tech_stack" "$target_file"; then + new_tech_entries+=("- $tech_stack ($CURRENT_BRANCH)") + fi + + if [[ -n "$NEW_DB" ]] && [[ "$NEW_DB" != "N/A" ]] && [[ "$NEW_DB" != "NEEDS CLARIFICATION" ]] && ! grep -q "$NEW_DB" "$target_file"; then + new_tech_entries+=("- $NEW_DB ($CURRENT_BRANCH)") + fi + + # Prepare new change entry + if [[ -n "$tech_stack" ]]; then + new_change_entry="- $CURRENT_BRANCH: Added $tech_stack" + elif [[ -n "$NEW_DB" ]] && [[ "$NEW_DB" != "N/A" ]] && [[ "$NEW_DB" != "NEEDS CLARIFICATION" ]]; then + new_change_entry="- $CURRENT_BRANCH: Added $NEW_DB" + fi + + # Check if sections exist in the file + local has_active_technologies=0 + local has_recent_changes=0 + + if grep -q "^## Active Technologies" "$target_file" 2>/dev/null; then + has_active_technologies=1 + fi + + if grep -q "^## Recent Changes" "$target_file" 2>/dev/null; then + has_recent_changes=1 + fi + + # Process file line by line + local in_tech_section=false + local in_changes_section=false + local tech_entries_added=false + local changes_entries_added=false + local existing_changes_count=0 + local file_ended=false + + while IFS= read -r line || [[ -n "$line" ]]; do + # Handle Active Technologies section + if [[ "$line" == "## Active Technologies" ]]; then + echo "$line" >> "$temp_file" + in_tech_section=true + continue + elif [[ $in_tech_section == true ]] && [[ "$line" =~ ^##[[:space:]] ]]; then + # Add new tech entries before closing the section + if [[ $tech_entries_added == false ]] && [[ ${#new_tech_entries[@]} -gt 0 ]]; then + printf '%s\n' "${new_tech_entries[@]}" >> "$temp_file" + tech_entries_added=true + fi + echo "$line" >> "$temp_file" + in_tech_section=false + continue + elif [[ $in_tech_section == true ]] && [[ -z "$line" ]]; then + # Add new tech entries before empty line in tech section + if [[ $tech_entries_added == false ]] && [[ ${#new_tech_entries[@]} -gt 0 ]]; then + printf '%s\n' "${new_tech_entries[@]}" >> "$temp_file" + tech_entries_added=true + fi + echo "$line" >> "$temp_file" + continue + fi + + # Handle Recent Changes section + if [[ "$line" == "## Recent Changes" ]]; then + echo "$line" >> "$temp_file" + # Add new change entry right after the heading + if [[ -n "$new_change_entry" ]]; then + echo "$new_change_entry" >> "$temp_file" + fi + in_changes_section=true + changes_entries_added=true + continue + elif [[ $in_changes_section == true ]] && [[ "$line" =~ ^##[[:space:]] ]]; then + echo "$line" >> "$temp_file" + in_changes_section=false + continue + elif [[ $in_changes_section == true ]] && [[ "$line" == "- "* ]]; then + # Keep only first 2 existing changes + if [[ $existing_changes_count -lt 2 ]]; then + echo "$line" >> "$temp_file" + ((existing_changes_count++)) + fi + continue + fi + + # Update timestamp + if [[ "$line" =~ (\*\*)?Last\ updated(\*\*)?:.*[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] ]]; then + echo "$line" | sed "s/[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]/$current_date/" >> "$temp_file" + else + echo "$line" >> "$temp_file" + fi + done < "$target_file" + + # Post-loop check: if we're still in the Active Technologies section and haven't added new entries + if [[ $in_tech_section == true ]] && [[ $tech_entries_added == false ]] && [[ ${#new_tech_entries[@]} -gt 0 ]]; then + printf '%s\n' "${new_tech_entries[@]}" >> "$temp_file" + tech_entries_added=true + fi + + # If sections don't exist, add them at the end of the file + if [[ $has_active_technologies -eq 0 ]] && [[ ${#new_tech_entries[@]} -gt 0 ]]; then + echo "" >> "$temp_file" + echo "## Active Technologies" >> "$temp_file" + printf '%s\n' "${new_tech_entries[@]}" >> "$temp_file" + tech_entries_added=true + fi + + if [[ $has_recent_changes -eq 0 ]] && [[ -n "$new_change_entry" ]]; then + echo "" >> "$temp_file" + echo "## Recent Changes" >> "$temp_file" + echo "$new_change_entry" >> "$temp_file" + changes_entries_added=true + fi + + # Ensure Cursor .mdc files have YAML frontmatter for auto-inclusion + if [[ "$target_file" == *.mdc ]]; then + if ! head -1 "$temp_file" | grep -q '^---'; then + local frontmatter_file + frontmatter_file=$(mktemp) || { rm -f "$temp_file"; return 1; } + printf '%s\n' "---" "description: Project Development Guidelines" "globs: [\"**/*\"]" "alwaysApply: true" "---" "" > "$frontmatter_file" + cat "$temp_file" >> "$frontmatter_file" + mv "$frontmatter_file" "$temp_file" + fi + fi + + # Move temp file to target atomically + if ! mv "$temp_file" "$target_file"; then + log_error "Failed to update target file" + rm -f "$temp_file" + return 1 + fi + + return 0 +} +#============================================================================== +# Main Agent File Update Function +#============================================================================== + +update_agent_file() { + local target_file="$1" + local agent_name="$2" + + if [[ -z "$target_file" ]] || [[ -z "$agent_name" ]]; then + log_error "update_agent_file requires target_file and agent_name parameters" + return 1 + fi + + log_info "Updating $agent_name context file: $target_file" + + local project_name + project_name=$(basename "$REPO_ROOT") + local current_date + current_date=$(date +%Y-%m-%d) + + # Create directory if it doesn't exist + local target_dir + target_dir=$(dirname "$target_file") + if [[ ! -d "$target_dir" ]]; then + if ! mkdir -p "$target_dir"; then + log_error "Failed to create directory: $target_dir" + return 1 + fi + fi + + if [[ ! -f "$target_file" ]]; then + # Create new file from template + local temp_file + temp_file=$(mktemp) || { + log_error "Failed to create temporary file" + return 1 + } + + if create_new_agent_file "$target_file" "$temp_file" "$project_name" "$current_date"; then + if mv "$temp_file" "$target_file"; then + log_success "Created new $agent_name context file" + else + log_error "Failed to move temporary file to $target_file" + rm -f "$temp_file" + return 1 + fi + else + log_error "Failed to create new agent file" + rm -f "$temp_file" + return 1 + fi + else + # Update existing file + if [[ ! -r "$target_file" ]]; then + log_error "Cannot read existing file: $target_file" + return 1 + fi + + if [[ ! -w "$target_file" ]]; then + log_error "Cannot write to existing file: $target_file" + return 1 + fi + + if update_existing_agent_file "$target_file" "$current_date"; then + log_success "Updated existing $agent_name context file" + else + log_error "Failed to update existing agent file" + return 1 + fi + fi + + return 0 +} + +#============================================================================== +# Agent Selection and Processing +#============================================================================== + +update_specific_agent() { + local agent_type="$1" + + case "$agent_type" in + claude) + update_agent_file "$CLAUDE_FILE" "Claude Code" || return 1 + ;; + gemini) + update_agent_file "$GEMINI_FILE" "Gemini CLI" || return 1 + ;; + copilot) + update_agent_file "$COPILOT_FILE" "GitHub Copilot" || return 1 + ;; + cursor-agent) + update_agent_file "$CURSOR_FILE" "Cursor IDE" || return 1 + ;; + qwen) + update_agent_file "$QWEN_FILE" "Qwen Code" || return 1 + ;; + opencode) + update_agent_file "$AGENTS_FILE" "opencode" || return 1 + ;; + codex) + update_agent_file "$AGENTS_FILE" "Codex CLI" || return 1 + ;; + windsurf) + update_agent_file "$WINDSURF_FILE" "Windsurf" || return 1 + ;; + kilocode) + update_agent_file "$KILOCODE_FILE" "Kilo Code" || return 1 + ;; + auggie) + update_agent_file "$AUGGIE_FILE" "Auggie CLI" || return 1 + ;; + roo) + update_agent_file "$ROO_FILE" "Roo Code" || return 1 + ;; + codebuddy) + update_agent_file "$CODEBUDDY_FILE" "CodeBuddy CLI" || return 1 + ;; + qodercli) + update_agent_file "$QODER_FILE" "Qoder CLI" || return 1 + ;; + amp) + update_agent_file "$AMP_FILE" "Amp" || return 1 + ;; + shai) + update_agent_file "$SHAI_FILE" "SHAI" || return 1 + ;; + tabnine) + update_agent_file "$TABNINE_FILE" "Tabnine CLI" || return 1 + ;; + kiro-cli) + update_agent_file "$KIRO_FILE" "Kiro CLI" || return 1 + ;; + agy) + update_agent_file "$AGY_FILE" "Antigravity" || return 1 + ;; + bob) + update_agent_file "$BOB_FILE" "IBM Bob" || return 1 + ;; + vibe) + update_agent_file "$VIBE_FILE" "Mistral Vibe" || return 1 + ;; + kimi) + update_agent_file "$KIMI_FILE" "Kimi Code" || return 1 + ;; + generic) + log_info "Generic agent: no predefined context file. Use the agent-specific update script for your agent." + ;; + *) + log_error "Unknown agent type '$agent_type'" + log_error "Expected: claude|gemini|copilot|cursor-agent|qwen|opencode|codex|windsurf|kilocode|auggie|roo|codebuddy|amp|shai|tabnine|kiro-cli|agy|bob|vibe|qodercli|kimi|generic" + exit 1 + ;; + esac +} + +update_all_existing_agents() { + local found_agent=false + local _updated_paths=() + + # Helper: skip non-existent files and files already updated (dedup by + # realpath so that variables pointing to the same file — e.g. AMP_FILE, + # KIRO_FILE, BOB_FILE all resolving to AGENTS_FILE — are only written once). + # Uses a linear array instead of associative array for bash 3.2 compatibility. + update_if_new() { + local file="$1" name="$2" + [[ -f "$file" ]] || return 0 + local real_path + real_path=$(realpath "$file" 2>/dev/null || echo "$file") + local p + if [[ ${#_updated_paths[@]} -gt 0 ]]; then + for p in "${_updated_paths[@]}"; do + [[ "$p" == "$real_path" ]] && return 0 + done + fi + update_agent_file "$file" "$name" || return 1 + _updated_paths+=("$real_path") + found_agent=true + } + + update_if_new "$CLAUDE_FILE" "Claude Code" + update_if_new "$GEMINI_FILE" "Gemini CLI" + update_if_new "$COPILOT_FILE" "GitHub Copilot" + update_if_new "$CURSOR_FILE" "Cursor IDE" + update_if_new "$QWEN_FILE" "Qwen Code" + update_if_new "$AGENTS_FILE" "Codex/opencode" + update_if_new "$AMP_FILE" "Amp" + update_if_new "$KIRO_FILE" "Kiro CLI" + update_if_new "$BOB_FILE" "IBM Bob" + update_if_new "$WINDSURF_FILE" "Windsurf" + update_if_new "$KILOCODE_FILE" "Kilo Code" + update_if_new "$AUGGIE_FILE" "Auggie CLI" + update_if_new "$ROO_FILE" "Roo Code" + update_if_new "$CODEBUDDY_FILE" "CodeBuddy CLI" + update_if_new "$SHAI_FILE" "SHAI" + update_if_new "$TABNINE_FILE" "Tabnine CLI" + update_if_new "$QODER_FILE" "Qoder CLI" + update_if_new "$AGY_FILE" "Antigravity" + update_if_new "$VIBE_FILE" "Mistral Vibe" + update_if_new "$KIMI_FILE" "Kimi Code" + + # If no agent files exist, create a default Claude file + if [[ "$found_agent" == false ]]; then + log_info "No existing agent files found, creating default Claude file..." + update_agent_file "$CLAUDE_FILE" "Claude Code" || return 1 + fi +} +print_summary() { + echo + log_info "Summary of changes:" + + if [[ -n "$NEW_LANG" ]]; then + echo " - Added language: $NEW_LANG" + fi + + if [[ -n "$NEW_FRAMEWORK" ]]; then + echo " - Added framework: $NEW_FRAMEWORK" + fi + + if [[ -n "$NEW_DB" ]] && [[ "$NEW_DB" != "N/A" ]]; then + echo " - Added database: $NEW_DB" + fi + + echo + log_info "Usage: $0 [claude|gemini|copilot|cursor-agent|qwen|opencode|codex|windsurf|kilocode|auggie|roo|codebuddy|amp|shai|tabnine|kiro-cli|agy|bob|vibe|qodercli|kimi|generic]" +} + +#============================================================================== +# Main Execution +#============================================================================== + +main() { + # Validate environment before proceeding + validate_environment + + log_info "=== Updating agent context files for feature $CURRENT_BRANCH ===" + + # Parse the plan file to extract project information + if ! parse_plan_data "$NEW_PLAN"; then + log_error "Failed to parse plan data" + exit 1 + fi + + # Process based on agent type argument + local success=true + + if [[ -z "$AGENT_TYPE" ]]; then + # No specific agent provided - update all existing agent files + log_info "No agent specified, updating all existing agent files..." + if ! update_all_existing_agents; then + success=false + fi + else + # Specific agent provided - update only that agent + log_info "Updating specific agent: $AGENT_TYPE" + if ! update_specific_agent "$AGENT_TYPE"; then + success=false + fi + fi + + # Print summary + print_summary + + if [[ "$success" == true ]]; then + log_success "Agent context update completed successfully" + exit 0 + else + log_error "Agent context update completed with errors" + exit 1 + fi +} + +# Execute main function if script is run directly +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi diff --git a/.specify/templates/agent-file-template.md b/.specify/templates/agent-file-template.md new file mode 100644 index 00000000..4cc7fd66 --- /dev/null +++ b/.specify/templates/agent-file-template.md @@ -0,0 +1,28 @@ +# [PROJECT NAME] Development Guidelines + +Auto-generated from all feature plans. Last updated: [DATE] + +## Active Technologies + +[EXTRACTED FROM ALL PLAN.MD FILES] + +## Project Structure + +```text +[ACTUAL STRUCTURE FROM PLANS] +``` + +## Commands + +[ONLY COMMANDS FOR ACTIVE TECHNOLOGIES] + +## Code Style + +[LANGUAGE-SPECIFIC, ONLY FOR LANGUAGES IN USE] + +## Recent Changes + +[LAST 3 FEATURES AND WHAT THEY ADDED] + + + diff --git a/.specify/templates/checklist-template.md b/.specify/templates/checklist-template.md new file mode 100644 index 00000000..806657da --- /dev/null +++ b/.specify/templates/checklist-template.md @@ -0,0 +1,40 @@ +# [CHECKLIST TYPE] Checklist: [FEATURE NAME] + +**Purpose**: [Brief description of what this checklist covers] +**Created**: [DATE] +**Feature**: [Link to spec.md or relevant documentation] + +**Note**: This checklist is generated by the `/speckit.checklist` command based on feature context and requirements. + + + +## [Category 1] + +- [ ] CHK001 First checklist item with clear action +- [ ] CHK002 Second checklist item +- [ ] CHK003 Third checklist item + +## [Category 2] + +- [ ] CHK004 Another category item +- [ ] CHK005 Item with specific criteria +- [ ] CHK006 Final item in this category + +## Notes + +- Check items off as completed: `[x]` +- Add comments or findings inline +- Link to relevant resources or documentation +- Items are numbered sequentially for easy reference diff --git a/.specify/templates/constitution-template.md b/.specify/templates/constitution-template.md new file mode 100644 index 00000000..a4670ff4 --- /dev/null +++ b/.specify/templates/constitution-template.md @@ -0,0 +1,50 @@ +# [PROJECT_NAME] Constitution + + +## Core Principles + +### [PRINCIPLE_1_NAME] + +[PRINCIPLE_1_DESCRIPTION] + + +### [PRINCIPLE_2_NAME] + +[PRINCIPLE_2_DESCRIPTION] + + +### [PRINCIPLE_3_NAME] + +[PRINCIPLE_3_DESCRIPTION] + + +### [PRINCIPLE_4_NAME] + +[PRINCIPLE_4_DESCRIPTION] + + +### [PRINCIPLE_5_NAME] + +[PRINCIPLE_5_DESCRIPTION] + + +## [SECTION_2_NAME] + + +[SECTION_2_CONTENT] + + +## [SECTION_3_NAME] + + +[SECTION_3_CONTENT] + + +## Governance + + +[GOVERNANCE_RULES] + + +**Version**: [CONSTITUTION_VERSION] | **Ratified**: [RATIFICATION_DATE] | **Last Amended**: [LAST_AMENDED_DATE] + diff --git a/.specify/templates/plan-template.md b/.specify/templates/plan-template.md new file mode 100644 index 00000000..5a2fafeb --- /dev/null +++ b/.specify/templates/plan-template.md @@ -0,0 +1,104 @@ +# Implementation Plan: [FEATURE] + +**Branch**: `[###-feature-name]` | **Date**: [DATE] | **Spec**: [link] +**Input**: Feature specification from `/specs/[###-feature-name]/spec.md` + +**Note**: This template is filled in by the `/speckit.plan` command. See `.specify/templates/plan-template.md` for the execution workflow. + +## Summary + +[Extract from feature spec: primary requirement + technical approach from research] + +## Technical Context + + + +**Language/Version**: [e.g., Python 3.11, Swift 5.9, Rust 1.75 or NEEDS CLARIFICATION] +**Primary Dependencies**: [e.g., FastAPI, UIKit, LLVM or NEEDS CLARIFICATION] +**Storage**: [if applicable, e.g., PostgreSQL, CoreData, files or N/A] +**Testing**: [e.g., pytest, XCTest, cargo test or NEEDS CLARIFICATION] +**Target Platform**: [e.g., Linux server, iOS 15+, WASM or NEEDS CLARIFICATION] +**Project Type**: [e.g., library/cli/web-service/mobile-app/compiler/desktop-app or NEEDS CLARIFICATION] +**Performance Goals**: [domain-specific, e.g., 1000 req/s, 10k lines/sec, 60 fps or NEEDS CLARIFICATION] +**Constraints**: [domain-specific, e.g., <200ms p95, <100MB memory, offline-capable or NEEDS CLARIFICATION] +**Scale/Scope**: [domain-specific, e.g., 10k users, 1M LOC, 50 screens or NEEDS CLARIFICATION] + +## Constitution Check + +*GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.* + +[Gates determined based on constitution file] + +## Project Structure + +### Documentation (this feature) + +```text +specs/[###-feature]/ +├── plan.md # This file (/speckit.plan command output) +├── research.md # Phase 0 output (/speckit.plan command) +├── data-model.md # Phase 1 output (/speckit.plan command) +├── quickstart.md # Phase 1 output (/speckit.plan command) +├── contracts/ # Phase 1 output (/speckit.plan command) +└── tasks.md # Phase 2 output (/speckit.tasks command - NOT created by /speckit.plan) +``` + +### Source Code (repository root) + + +```text +# [REMOVE IF UNUSED] Option 1: Single project (DEFAULT) +src/ +├── models/ +├── services/ +├── cli/ +└── lib/ + +tests/ +├── contract/ +├── integration/ +└── unit/ + +# [REMOVE IF UNUSED] Option 2: Web application (when "frontend" + "backend" detected) +backend/ +├── src/ +│ ├── models/ +│ ├── services/ +│ └── api/ +└── tests/ + +frontend/ +├── src/ +│ ├── components/ +│ ├── pages/ +│ └── services/ +└── tests/ + +# [REMOVE IF UNUSED] Option 3: Mobile + API (when "iOS/Android" detected) +api/ +└── [same as backend above] + +ios/ or android/ +└── [platform-specific structure: feature modules, UI flows, platform tests] +``` + +**Structure Decision**: [Document the selected structure and reference the real +directories captured above] + +## Complexity Tracking + +> **Fill ONLY if Constitution Check has violations that must be justified** + +| Violation | Why Needed | Simpler Alternative Rejected Because | +|-----------|------------|-------------------------------------| +| [e.g., 4th project] | [current need] | [why 3 projects insufficient] | +| [e.g., Repository pattern] | [specific problem] | [why direct DB access insufficient] | diff --git a/.specify/templates/spec-template.md b/.specify/templates/spec-template.md new file mode 100644 index 00000000..c67d9149 --- /dev/null +++ b/.specify/templates/spec-template.md @@ -0,0 +1,115 @@ +# Feature Specification: [FEATURE NAME] + +**Feature Branch**: `[###-feature-name]` +**Created**: [DATE] +**Status**: Draft +**Input**: User description: "$ARGUMENTS" + +## User Scenarios & Testing *(mandatory)* + + + +### User Story 1 - [Brief Title] (Priority: P1) + +[Describe this user journey in plain language] + +**Why this priority**: [Explain the value and why it has this priority level] + +**Independent Test**: [Describe how this can be tested independently - e.g., "Can be fully tested by [specific action] and delivers [specific value]"] + +**Acceptance Scenarios**: + +1. **Given** [initial state], **When** [action], **Then** [expected outcome] +2. **Given** [initial state], **When** [action], **Then** [expected outcome] + +--- + +### User Story 2 - [Brief Title] (Priority: P2) + +[Describe this user journey in plain language] + +**Why this priority**: [Explain the value and why it has this priority level] + +**Independent Test**: [Describe how this can be tested independently] + +**Acceptance Scenarios**: + +1. **Given** [initial state], **When** [action], **Then** [expected outcome] + +--- + +### User Story 3 - [Brief Title] (Priority: P3) + +[Describe this user journey in plain language] + +**Why this priority**: [Explain the value and why it has this priority level] + +**Independent Test**: [Describe how this can be tested independently] + +**Acceptance Scenarios**: + +1. **Given** [initial state], **When** [action], **Then** [expected outcome] + +--- + +[Add more user stories as needed, each with an assigned priority] + +### Edge Cases + + + +- What happens when [boundary condition]? +- How does system handle [error scenario]? + +## Requirements *(mandatory)* + + + +### Functional Requirements + +- **FR-001**: System MUST [specific capability, e.g., "allow users to create accounts"] +- **FR-002**: System MUST [specific capability, e.g., "validate email addresses"] +- **FR-003**: Users MUST be able to [key interaction, e.g., "reset their password"] +- **FR-004**: System MUST [data requirement, e.g., "persist user preferences"] +- **FR-005**: System MUST [behavior, e.g., "log all security events"] + +*Example of marking unclear requirements:* + +- **FR-006**: System MUST authenticate users via [NEEDS CLARIFICATION: auth method not specified - email/password, SSO, OAuth?] +- **FR-007**: System MUST retain user data for [NEEDS CLARIFICATION: retention period not specified] + +### Key Entities *(include if feature involves data)* + +- **[Entity 1]**: [What it represents, key attributes without implementation] +- **[Entity 2]**: [What it represents, relationships to other entities] + +## Success Criteria *(mandatory)* + + + +### Measurable Outcomes + +- **SC-001**: [Measurable metric, e.g., "Users can complete account creation in under 2 minutes"] +- **SC-002**: [Measurable metric, e.g., "System handles 1000 concurrent users without degradation"] +- **SC-003**: [User satisfaction metric, e.g., "90% of users successfully complete primary task on first attempt"] +- **SC-004**: [Business metric, e.g., "Reduce support tickets related to [X] by 50%"] diff --git a/.specify/templates/tasks-template.md b/.specify/templates/tasks-template.md new file mode 100644 index 00000000..60f9be45 --- /dev/null +++ b/.specify/templates/tasks-template.md @@ -0,0 +1,251 @@ +--- + +description: "Task list template for feature implementation" +--- + +# Tasks: [FEATURE NAME] + +**Input**: Design documents from `/specs/[###-feature-name]/` +**Prerequisites**: plan.md (required), spec.md (required for user stories), research.md, data-model.md, contracts/ + +**Tests**: The examples below include test tasks. Tests are OPTIONAL - only include them if explicitly requested in the feature specification. + +**Organization**: Tasks are grouped by user story to enable independent implementation and testing of each story. + +## Format: `[ID] [P?] [Story] Description` + +- **[P]**: Can run in parallel (different files, no dependencies) +- **[Story]**: Which user story this task belongs to (e.g., US1, US2, US3) +- Include exact file paths in descriptions + +## Path Conventions + +- **Single project**: `src/`, `tests/` at repository root +- **Web app**: `backend/src/`, `frontend/src/` +- **Mobile**: `api/src/`, `ios/src/` or `android/src/` +- Paths shown below assume single project - adjust based on plan.md structure + + + +## Phase 1: Setup (Shared Infrastructure) + +**Purpose**: Project initialization and basic structure + +- [ ] T001 Create project structure per implementation plan +- [ ] T002 Initialize [language] project with [framework] dependencies +- [ ] T003 [P] Configure linting and formatting tools + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +**Purpose**: Core infrastructure that MUST be complete before ANY user story can be implemented + +**⚠️ CRITICAL**: No user story work can begin until this phase is complete + +Examples of foundational tasks (adjust based on your project): + +- [ ] T004 Setup database schema and migrations framework +- [ ] T005 [P] Implement authentication/authorization framework +- [ ] T006 [P] Setup API routing and middleware structure +- [ ] T007 Create base models/entities that all stories depend on +- [ ] T008 Configure error handling and logging infrastructure +- [ ] T009 Setup environment configuration management + +**Checkpoint**: Foundation ready - user story implementation can now begin in parallel + +--- + +## Phase 3: User Story 1 - [Title] (Priority: P1) 🎯 MVP + +**Goal**: [Brief description of what this story delivers] + +**Independent Test**: [How to verify this story works on its own] + +### Tests for User Story 1 (OPTIONAL - only if tests requested) ⚠️ + +> **NOTE: Write these tests FIRST, ensure they FAIL before implementation** + +- [ ] T010 [P] [US1] Contract test for [endpoint] in tests/contract/test_[name].py +- [ ] T011 [P] [US1] Integration test for [user journey] in tests/integration/test_[name].py + +### Implementation for User Story 1 + +- [ ] T012 [P] [US1] Create [Entity1] model in src/models/[entity1].py +- [ ] T013 [P] [US1] Create [Entity2] model in src/models/[entity2].py +- [ ] T014 [US1] Implement [Service] in src/services/[service].py (depends on T012, T013) +- [ ] T015 [US1] Implement [endpoint/feature] in src/[location]/[file].py +- [ ] T016 [US1] Add validation and error handling +- [ ] T017 [US1] Add logging for user story 1 operations + +**Checkpoint**: At this point, User Story 1 should be fully functional and testable independently + +--- + +## Phase 4: User Story 2 - [Title] (Priority: P2) + +**Goal**: [Brief description of what this story delivers] + +**Independent Test**: [How to verify this story works on its own] + +### Tests for User Story 2 (OPTIONAL - only if tests requested) ⚠️ + +- [ ] T018 [P] [US2] Contract test for [endpoint] in tests/contract/test_[name].py +- [ ] T019 [P] [US2] Integration test for [user journey] in tests/integration/test_[name].py + +### Implementation for User Story 2 + +- [ ] T020 [P] [US2] Create [Entity] model in src/models/[entity].py +- [ ] T021 [US2] Implement [Service] in src/services/[service].py +- [ ] T022 [US2] Implement [endpoint/feature] in src/[location]/[file].py +- [ ] T023 [US2] Integrate with User Story 1 components (if needed) + +**Checkpoint**: At this point, User Stories 1 AND 2 should both work independently + +--- + +## Phase 5: User Story 3 - [Title] (Priority: P3) + +**Goal**: [Brief description of what this story delivers] + +**Independent Test**: [How to verify this story works on its own] + +### Tests for User Story 3 (OPTIONAL - only if tests requested) ⚠️ + +- [ ] T024 [P] [US3] Contract test for [endpoint] in tests/contract/test_[name].py +- [ ] T025 [P] [US3] Integration test for [user journey] in tests/integration/test_[name].py + +### Implementation for User Story 3 + +- [ ] T026 [P] [US3] Create [Entity] model in src/models/[entity].py +- [ ] T027 [US3] Implement [Service] in src/services/[service].py +- [ ] T028 [US3] Implement [endpoint/feature] in src/[location]/[file].py + +**Checkpoint**: All user stories should now be independently functional + +--- + +[Add more user story phases as needed, following the same pattern] + +--- + +## Phase N: Polish & Cross-Cutting Concerns + +**Purpose**: Improvements that affect multiple user stories + +- [ ] TXXX [P] Documentation updates in docs/ +- [ ] TXXX Code cleanup and refactoring +- [ ] TXXX Performance optimization across all stories +- [ ] TXXX [P] Additional unit tests (if requested) in tests/unit/ +- [ ] TXXX Security hardening +- [ ] TXXX Run quickstart.md validation + +--- + +## Dependencies & Execution Order + +### Phase Dependencies + +- **Setup (Phase 1)**: No dependencies - can start immediately +- **Foundational (Phase 2)**: Depends on Setup completion - BLOCKS all user stories +- **User Stories (Phase 3+)**: All depend on Foundational phase completion + - User stories can then proceed in parallel (if staffed) + - Or sequentially in priority order (P1 → P2 → P3) +- **Polish (Final Phase)**: Depends on all desired user stories being complete + +### User Story Dependencies + +- **User Story 1 (P1)**: Can start after Foundational (Phase 2) - No dependencies on other stories +- **User Story 2 (P2)**: Can start after Foundational (Phase 2) - May integrate with US1 but should be independently testable +- **User Story 3 (P3)**: Can start after Foundational (Phase 2) - May integrate with US1/US2 but should be independently testable + +### Within Each User Story + +- Tests (if included) MUST be written and FAIL before implementation +- Models before services +- Services before endpoints +- Core implementation before integration +- Story complete before moving to next priority + +### Parallel Opportunities + +- All Setup tasks marked [P] can run in parallel +- All Foundational tasks marked [P] can run in parallel (within Phase 2) +- Once Foundational phase completes, all user stories can start in parallel (if team capacity allows) +- All tests for a user story marked [P] can run in parallel +- Models within a story marked [P] can run in parallel +- Different user stories can be worked on in parallel by different team members + +--- + +## Parallel Example: User Story 1 + +```bash +# Launch all tests for User Story 1 together (if tests requested): +Task: "Contract test for [endpoint] in tests/contract/test_[name].py" +Task: "Integration test for [user journey] in tests/integration/test_[name].py" + +# Launch all models for User Story 1 together: +Task: "Create [Entity1] model in src/models/[entity1].py" +Task: "Create [Entity2] model in src/models/[entity2].py" +``` + +--- + +## Implementation Strategy + +### MVP First (User Story 1 Only) + +1. Complete Phase 1: Setup +2. Complete Phase 2: Foundational (CRITICAL - blocks all stories) +3. Complete Phase 3: User Story 1 +4. **STOP and VALIDATE**: Test User Story 1 independently +5. Deploy/demo if ready + +### Incremental Delivery + +1. Complete Setup + Foundational → Foundation ready +2. Add User Story 1 → Test independently → Deploy/Demo (MVP!) +3. Add User Story 2 → Test independently → Deploy/Demo +4. Add User Story 3 → Test independently → Deploy/Demo +5. Each story adds value without breaking previous stories + +### Parallel Team Strategy + +With multiple developers: + +1. Team completes Setup + Foundational together +2. Once Foundational is done: + - Developer A: User Story 1 + - Developer B: User Story 2 + - Developer C: User Story 3 +3. Stories complete and integrate independently + +--- + +## Notes + +- [P] tasks = different files, no dependencies +- [Story] label maps task to specific user story for traceability +- Each user story should be independently completable and testable +- Verify tests fail before implementing +- Commit after each task or logical group +- Stop at any checkpoint to validate story independently +- Avoid: vague tasks, same file conflicts, cross-story dependencies that break independence From f7ce8cda01db79835bf317585db4d990bcec96c7 Mon Sep 17 00:00:00 2001 From: Brian Gebel Date: Sat, 14 Mar 2026 20:24:56 -0700 Subject: [PATCH 3/7] fix(ci): harden CI/CD workflows and test compatibility for plugin system CI/CD workflow fixes: - e2e-tests.yml: add FalkorDB service container with health check, bump checkout@v3->v4 and setup-python@v4->v5, add cache: pip, expose FALKORDB_HOST/FALKORDB_PORT/DATABASE_TYPE env vars for test runner - macos.yml: replace || true shell suppression with continue-on-error: true on Install system deps, Run index, and Try find steps - test-plugins.yml: remove silent pip fallback from core install step so broken installs fail fast instead of silently proceeding with partial deps - test.yml: add cache: pip to setup-python step Application fixes (from python-pro): - cgc.spec: PyInstaller frozen binary fixes - tests/integration/plugin/test_otel_integration.py: asyncio fix - tests/integration/plugin/test_memory_integration.py: importorskip guards - tests/unit/plugin/test_otel_processor.py: importorskip guards - tests/unit/plugin/test_xdebug_parser.py: importorskip guards - plugins/cgc-plugin-{stub,otel,xdebug,memory}/README.md: plugin READMEs - pyinstaller_hooks/: PyInstaller runtime hooks for plugin discovery Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/e2e-tests.yml | 20 +++++++- .github/workflows/macos.yml | 9 ++-- .github/workflows/test-plugins.yml | 2 +- .github/workflows/test.yml | 1 + cgc.spec | 39 ++++++++++++-- plugins/cgc-plugin-memory/README.md | 46 +++++++++++++++++ plugins/cgc-plugin-otel/README.md | 45 ++++++++++++++++ plugins/cgc-plugin-stub/README.md | 36 +++++++++++++ plugins/cgc-plugin-xdebug/README.md | 50 ++++++++++++++++++ .../hook-codegraphcontext.plugin_registry.py | 51 +++++++++++++++++++ .../rthook_importlib_metadata.py | 46 +++++++++++++++++ .../plugin/test_memory_integration.py | 7 +-- .../plugin/test_otel_integration.py | 48 +++++++++-------- tests/unit/plugin/test_otel_processor.py | 8 +-- tests/unit/plugin/test_xdebug_parser.py | 7 +-- 15 files changed, 371 insertions(+), 44 deletions(-) create mode 100644 plugins/cgc-plugin-memory/README.md create mode 100644 plugins/cgc-plugin-otel/README.md create mode 100644 plugins/cgc-plugin-stub/README.md create mode 100644 plugins/cgc-plugin-xdebug/README.md create mode 100644 pyinstaller_hooks/hook-codegraphcontext.plugin_registry.py create mode 100644 pyinstaller_hooks/rthook_importlib_metadata.py diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 2907fbab..a07e2d4f 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -10,14 +10,26 @@ jobs: test: runs-on: ubuntu-latest + services: + falkordb: + image: falkordb/falkordb:latest + ports: + - 6379:6379 + options: >- + --health-cmd "redis-cli ping" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: - name: Check out code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: '3.12' + cache: 'pip' - name: Install dependencies run: | @@ -26,6 +38,10 @@ jobs: pip install pytest - name: Run end-to-end tests + env: + FALKORDB_HOST: localhost + FALKORDB_PORT: 6379 + DATABASE_TYPE: falkordb-remote run: | chmod +x tests/run_tests.sh ./tests/run_tests.sh e2e diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 5ee4297c..f62dabe9 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -19,9 +19,10 @@ jobs: python-version: "3.12" - name: Install system deps + continue-on-error: true run: | brew update - brew install ripgrep || true + brew install ripgrep - name: Install CodeGraphContext run: | @@ -35,12 +36,14 @@ jobs: df -h - name: Run index (verbose) + continue-on-error: true run: | - cgc index -f --debug || true + cgc index -f --debug - name: Try find + continue-on-error: true run: | - cgc find content "def" --debug || true + cgc find content "def" --debug - name: Upload logs if: always() diff --git a/.github/workflows/test-plugins.yml b/.github/workflows/test-plugins.yml index 8945ece3..83817a52 100644 --- a/.github/workflows/test-plugins.yml +++ b/.github/workflows/test-plugins.yml @@ -32,7 +32,7 @@ jobs: - name: Install core CGC (no extras) and dev dependencies run: | pip install --no-cache-dir packaging pytest pytest-mock - pip install --no-cache-dir -e ".[dev]" || pip install --no-cache-dir packaging pytest pytest-mock + pip install --no-cache-dir -e ".[dev]" - name: Install stub plugin (editable) run: pip install --no-cache-dir -e plugins/cgc-plugin-stub diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4429a088..807342eb 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,6 +21,7 @@ jobs: uses: actions/setup-python@v5 with: python-version: "3.12" + cache: 'pip' - name: Install dependencies run: | diff --git a/cgc.spec b/cgc.spec index 8b2a8473..d01deff6 100644 --- a/cgc.spec +++ b/cgc.spec @@ -5,7 +5,7 @@ import sys import os from pathlib import Path -from PyInstaller.utils.hooks import collect_data_files, collect_submodules, collect_all +from PyInstaller.utils.hooks import collect_data_files, collect_submodules, collect_all, collect_entry_point block_cipher = None @@ -138,14 +138,47 @@ hidden_imports = [ 'httpx', 'httpcore', 'importlib', + 'importlib.metadata', + 'importlib.metadata._meta', + 'importlib.metadata._adapters', + 'importlib.metadata._itertools', + 'importlib.metadata._functools', + 'importlib.metadata._text', 'asyncio', 'pkg_resources', + 'pkg_resources.extern', 'threading', 'subprocess', 'socket', 'atexit', + # plugin_registry.py discovers plugins via importlib.metadata.entry_points(); + # each installed plugin's distribution metadata must be bundled so that + # entry_points(group=...) resolves correctly in a frozen executable. + 'codegraphcontext.plugin_registry', ] +# ── Plugin entry-point metadata collection ──────────────────────────────── +# PyInstaller cannot discover entry points at freeze time unless the +# distribution METADATA / entry_points.txt files are explicitly copied into +# the bundle. collect_entry_point() returns (datas, hidden_imports) for +# every distribution that declares the requested group. +_plugin_ep_groups = ['cgc_cli_plugins', 'cgc_mcp_plugins'] +for _ep_group in _plugin_ep_groups: + try: + _ep_datas, _ep_hidden = collect_entry_point(_ep_group) + datas += _ep_datas + hidden_imports += _ep_hidden + except Exception as _ep_exc: + print(f"Warning: collect_entry_point('{_ep_group}') failed: {_ep_exc}") + +# Bundle the codegraphcontext distribution metadata so that +# importlib.metadata.version("codegraphcontext") resolves in the frozen binary +# and PluginRegistry._get_cgc_version() returns the correct version string. +try: + datas += collect_data_files('codegraphcontext', includes=['**/*.dist-info/**/*']) +except Exception as _cgc_meta_exc: + print(f"Warning: collect_data_files('codegraphcontext') failed: {_cgc_meta_exc}") + # Bin extensions by platform ext = '*.so' @@ -267,9 +300,9 @@ a = Analysis( binaries=binaries, datas=datas, hiddenimports=hidden_imports, - hookspath=[], + hookspath=['pyinstaller_hooks'], hooksconfig={}, - runtime_hooks=[], + runtime_hooks=['pyinstaller_hooks/rthook_importlib_metadata.py'], excludes=[ 'tkinter', '_tkinter', 'matplotlib', 'numpy', 'pandas', 'scipy', 'PIL', 'cv2', 'torch', 'tensorflow', 'jupyter', 'notebook', 'IPython', diff --git a/plugins/cgc-plugin-memory/README.md b/plugins/cgc-plugin-memory/README.md new file mode 100644 index 00000000..6345d6c1 --- /dev/null +++ b/plugins/cgc-plugin-memory/README.md @@ -0,0 +1,46 @@ +# cgc-plugin-memory + +Project knowledge memory plugin for [CodeGraphContext](https://github.com/CodeGraphContext/CodeGraphContext). + +## Overview + +This plugin provides a persistent, searchable memory layer for project-level knowledge +stored in the CGC Neo4j graph. It allows AI assistants and developers to store +specifications, notes, and design decisions as `Memory` nodes, link them to specific +code graph entities (classes, functions, files), and retrieve them via full-text search. + +## Features + +- Store arbitrary knowledge as typed `Memory` nodes (spec, note, decision, etc.) +- Link memory entries to code graph nodes using `DESCRIBES` relationships +- Full-text search across stored memory via a Neo4j fulltext index +- Query undocumented code nodes (classes, functions without any linked memory) +- Exposes a `memory` CLI command group and MCP tools prefixed with `memory_` + +## Requirements + +- Python 3.10+ +- CodeGraphContext >= 0.3.0 +- Neo4j >= 5.15 + +## Installation + +```bash +pip install -e plugins/cgc-plugin-memory +``` + +## MCP tools + +| Tool | Description | +|---|---| +| `memory_store` | Persist a knowledge entry, optionally linking it to a code node | +| `memory_search` | Full-text search across stored memory entries | +| `memory_undocumented` | List code nodes that have no linked memory entries | +| `memory_link` | Create a `DESCRIBES` edge between an existing memory entry and a code node | + +## Entry points + +| Group | Name | Target | +|---|---|---| +| `cgc_cli_plugins` | `memory` | `cgc_plugin_memory.cli:get_plugin_commands` | +| `cgc_mcp_plugins` | `memory` | `cgc_plugin_memory.mcp_tools:get_mcp_tools` | diff --git a/plugins/cgc-plugin-otel/README.md b/plugins/cgc-plugin-otel/README.md new file mode 100644 index 00000000..b0b381a6 --- /dev/null +++ b/plugins/cgc-plugin-otel/README.md @@ -0,0 +1,45 @@ +# cgc-plugin-otel + +OpenTelemetry span processor plugin for [CodeGraphContext](https://github.com/CodeGraphContext/CodeGraphContext). + +## Overview + +This plugin ingests OpenTelemetry spans from PHP services (e.g. Laravel) via a gRPC +OTLP receiver and writes them into the CGC Neo4j graph. Spans are correlated with +code graph nodes (classes, methods) using `CORRELATES_TO` relationships, enabling +cross-layer queries that link runtime traces to static code structure. + +## Features + +- gRPC OTLP receiver listening on port 5317 +- Extracts PHP context from OTel span attributes (`code.namespace`, `code.function`, etc.) +- Writes `Service`, `Trace`, and `Span` nodes to Neo4j +- Creates `PART_OF`, `CHILD_OF`, `CORRELATES_TO`, and `CALLS_SERVICE` relationships +- Dead-letter queue (DLQ) for spans that fail to persist +- Exposes a `otel` CLI command group and MCP tools prefixed with `otel_` + +## Requirements + +- Python 3.10+ +- CodeGraphContext >= 0.3.0 +- Neo4j >= 5.15 +- grpcio >= 1.57 +- opentelemetry-sdk >= 1.20 + +## Installation + +```bash +pip install -e plugins/cgc-plugin-otel +``` + +## MCP tool naming + +All MCP tools contributed by this plugin are prefixed with `otel_` +(e.g. `otel_query_spans`). + +## Entry points + +| Group | Name | Target | +|---|---|---| +| `cgc_cli_plugins` | `otel` | `cgc_plugin_otel.cli:get_plugin_commands` | +| `cgc_mcp_plugins` | `otel` | `cgc_plugin_otel.mcp_tools:get_mcp_tools` | diff --git a/plugins/cgc-plugin-stub/README.md b/plugins/cgc-plugin-stub/README.md new file mode 100644 index 00000000..ccd2c448 --- /dev/null +++ b/plugins/cgc-plugin-stub/README.md @@ -0,0 +1,36 @@ +# cgc-plugin-stub + +Minimal stub plugin for testing the CGC plugin extension system. + +## Overview + +This package is a reference fixture used by the CGC test suite to exercise plugin +discovery, registration, and lifecycle without requiring any real infrastructure. +It implements the minimum required interface (`PLUGIN_METADATA`, `get_plugin_commands()`, +`get_mcp_tools()`, `get_mcp_handlers()`) and contributes a no-op `stub` CLI command +group and a single `stub_ping` MCP tool. + +## Usage + +Install for development to enable the plugin integration and E2E test suites: + +```bash +pip install -e plugins/cgc-plugin-stub +``` + +Then run: + +```bash +PYTHONPATH=src pytest tests/unit/plugin/ tests/integration/plugin/ tests/e2e/plugin/ -v +``` + +## Entry points + +| Group | Name | Target | +|---|---|---| +| `cgc_cli_plugins` | `stub` | `cgc_plugin_stub.cli:get_plugin_commands` | +| `cgc_mcp_plugins` | `stub` | `cgc_plugin_stub.mcp_tools:get_mcp_tools` | + +## Note + +This plugin is not intended for production use. It exists solely as a test fixture. diff --git a/plugins/cgc-plugin-xdebug/README.md b/plugins/cgc-plugin-xdebug/README.md new file mode 100644 index 00000000..fb2f7c3d --- /dev/null +++ b/plugins/cgc-plugin-xdebug/README.md @@ -0,0 +1,50 @@ +# cgc-plugin-xdebug + +Xdebug DBGp call-stack listener plugin for [CodeGraphContext](https://github.com/CodeGraphContext/CodeGraphContext). + +**Intended for development and staging environments only — do not enable in production.** + +## Overview + +This plugin listens for Xdebug DBGp protocol connections on port 9003 and captures +PHP call-stack frames in real time. Parsed frames are written to the CGC Neo4j graph +as `CallChain` nodes, correlated with existing code graph nodes via their fully-qualified +names. This enables live execution path analysis alongside static code structure. + +## Features + +- TCP server accepting Xdebug DBGp connections on port 9003 +- Parses `stack_get` XML responses into structured frame dicts +- Computes deterministic `chain_hash` for deduplicating identical call chains +- Writes call-stack data to Neo4j as `CallChain` / `CallFrame` nodes +- Exposes an `xdebug` CLI command group and MCP tools prefixed with `xdebug_` + +## Requirements + +- Python 3.10+ +- CodeGraphContext >= 0.3.0 +- Neo4j >= 5.15 +- Xdebug configured with `xdebug.mode=debug` and `xdebug.client_host` pointing at the CGC host + +## Installation + +```bash +pip install -e plugins/cgc-plugin-xdebug +``` + +## Runtime activation + +Set the environment variable `CGC_PLUGIN_XDEBUG_ENABLED=true` before starting the +plugin server, otherwise the DBGp listener will not start. + +## MCP tool naming + +All MCP tools contributed by this plugin are prefixed with `xdebug_` +(e.g. `xdebug_query_callchain`). + +## Entry points + +| Group | Name | Target | +|---|---|---| +| `cgc_cli_plugins` | `xdebug` | `cgc_plugin_xdebug.cli:get_plugin_commands` | +| `cgc_mcp_plugins` | `xdebug` | `cgc_plugin_xdebug.mcp_tools:get_mcp_tools` | diff --git a/pyinstaller_hooks/hook-codegraphcontext.plugin_registry.py b/pyinstaller_hooks/hook-codegraphcontext.plugin_registry.py new file mode 100644 index 00000000..e6963cd5 --- /dev/null +++ b/pyinstaller_hooks/hook-codegraphcontext.plugin_registry.py @@ -0,0 +1,51 @@ +# PyInstaller hook for codegraphcontext.plugin_registry +# +# plugin_registry.py uses importlib.metadata.entry_points(group=...) to +# discover installed CGC plugins at runtime. For this to work in a frozen +# binary the distribution METADATA (including entry_points.txt) for every +# relevant package must be bundled. +# +# This hook: +# 1. Collects the codegraphcontext distribution metadata so the core +# package's own entry points are resolvable in the frozen binary. +# 2. Declares importlib.metadata internals as hidden imports to ensure +# the metadata resolution machinery is included. + +from PyInstaller.utils.hooks import collect_data_files, collect_entry_point + +datas = [] +hiddenimports = [ + "importlib.metadata", + "importlib.metadata._meta", + "importlib.metadata._adapters", + "importlib.metadata._itertools", + "importlib.metadata._functools", + "importlib.metadata._text", + "importlib.metadata.compat.functools", + "importlib.metadata.compat.py39", + "pkg_resources", + "pkg_resources.extern", +] + +# Bundle the codegraphcontext package distribution metadata so that +# importlib.metadata.version("codegraphcontext") resolves inside the frozen +# binary and PluginRegistry._get_cgc_version() returns the correct version. +try: + datas += collect_data_files("codegraphcontext", includes=["**/*.dist-info/**/*"]) +except Exception: + pass + +# Collect distribution METADATA for both plugin entry-point groups so that +# entry_points(group="cgc_cli_plugins") and entry_points(group="cgc_mcp_plugins") +# resolve correctly for any plugin that is installed at freeze time. +for _group in ("cgc_cli_plugins", "cgc_mcp_plugins"): + try: + _ep_datas, _ep_hidden = collect_entry_point(_group) + datas += _ep_datas + hiddenimports += _ep_hidden + except Exception as exc: + import warnings + warnings.warn( + f"hook-codegraphcontext.plugin_registry: collect_entry_point('{_group}') " + f"failed: {exc}" + ) diff --git a/pyinstaller_hooks/rthook_importlib_metadata.py b/pyinstaller_hooks/rthook_importlib_metadata.py new file mode 100644 index 00000000..1a9f9253 --- /dev/null +++ b/pyinstaller_hooks/rthook_importlib_metadata.py @@ -0,0 +1,46 @@ +# Runtime hook: ensure importlib.metadata can resolve entry points in a +# PyInstaller one-file frozen executable. +# +# When the frozen binary unpacks into sys._MEIPASS, distribution METADATA +# directories land there. importlib.metadata uses PathDistribution finders +# that walk sys.path. PyInstaller already inserts _MEIPASS at sys.path[0], +# but the metadata sub-directories are nested under site-packages-style paths. +# This hook adds the _MEIPASS path explicitly so entry_points(group=...) works. +# +# It also registers a fallback using pkg_resources so that any code path that +# calls pkg_resources.iter_entry_points() also resolves correctly. + +import sys +import os + +_meipass = getattr(sys, "_MEIPASS", None) + +if _meipass: + # Ensure _MEIPASS is in sys.path for importlib.metadata path finders. + if _meipass not in sys.path: + sys.path.insert(0, _meipass) + + # Force pkg_resources to rescan working_set so entry points registered + # via .dist-info/entry_points.txt inside _MEIPASS are visible. + try: + import pkg_resources + pkg_resources._initialize_master_working_set() + except Exception: + pass + + # Patch importlib.metadata to also search _MEIPASS for distributions. + try: + from importlib.metadata import MetadataPathFinder + import importlib.metadata as _ilm + + _orig_search_paths = getattr(_ilm, "_search_paths", None) + + def _patched_search_paths(name): # type: ignore[override] + paths = [_meipass] + if _orig_search_paths is not None: + paths.extend(_orig_search_paths(name)) + return paths + + _ilm._search_paths = _patched_search_paths + except Exception: + pass diff --git a/tests/integration/plugin/test_memory_integration.py b/tests/integration/plugin/test_memory_integration.py index 4b0e92c8..07601e0b 100644 --- a/tests/integration/plugin/test_memory_integration.py +++ b/tests/integration/plugin/test_memory_integration.py @@ -6,12 +6,13 @@ """ from __future__ import annotations -import sys -import os import pytest from unittest.mock import MagicMock, call -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../../plugins/cgc-plugin-memory/src")) +cgc_plugin_memory = pytest.importorskip( + "cgc_plugin_memory", + reason="cgc-plugin-memory is not installed; skipping memory integration tests", +) # --------------------------------------------------------------------------- diff --git a/tests/integration/plugin/test_otel_integration.py b/tests/integration/plugin/test_otel_integration.py index a5c9f67b..6236469e 100644 --- a/tests/integration/plugin/test_otel_integration.py +++ b/tests/integration/plugin/test_otel_integration.py @@ -7,13 +7,13 @@ """ from __future__ import annotations -import asyncio -import sys -import os import pytest from unittest.mock import AsyncMock, MagicMock, call, patch -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../../plugins/cgc-plugin-otel/src")) +cgc_plugin_otel = pytest.importorskip( + "cgc_plugin_otel", + reason="cgc-plugin-otel is not installed; skipping otel integration tests", +) # --------------------------------------------------------------------------- @@ -72,109 +72,107 @@ def _make_db_manager(): # Tests # --------------------------------------------------------------------------- +@pytest.mark.asyncio class TestAsyncOtelWriterBatch: - def _run(self, coro): - return asyncio.run(coro) - - def test_write_batch_issues_merge_service(self): + async def test_write_batch_issues_merge_service(self): """write_batch() issues a MERGE for the Service node.""" from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter db_manager, session = _make_db_manager() writer = AsyncOtelWriter(db_manager) span = _make_span() - self._run(writer.write_batch([span])) + await writer.write_batch([span]) cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] assert any("MERGE" in c and "Service" in c for c in cypher_calls), \ f"No Service MERGE found in calls: {cypher_calls}" - def test_write_batch_issues_merge_span(self): + async def test_write_batch_issues_merge_span(self): """write_batch() issues a MERGE for the Span node.""" from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter db_manager, session = _make_db_manager() writer = AsyncOtelWriter(db_manager) span = _make_span() - self._run(writer.write_batch([span])) + await writer.write_batch([span]) cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] assert any("MERGE" in c and "Span" in c for c in cypher_calls) - def test_write_batch_links_span_to_trace(self): + async def test_write_batch_links_span_to_trace(self): """write_batch() creates a PART_OF relationship between Span and Trace.""" from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter db_manager, session = _make_db_manager() writer = AsyncOtelWriter(db_manager) span = _make_span() - self._run(writer.write_batch([span])) + await writer.write_batch([span]) cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] assert any("PART_OF" in c for c in cypher_calls) - def test_write_batch_creates_child_of_for_parent_span_id(self): + async def test_write_batch_creates_child_of_for_parent_span_id(self): """CHILD_OF relationship is created when parent_span_id is set.""" from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter db_manager, session = _make_db_manager() writer = AsyncOtelWriter(db_manager) span = _make_span(span_id="child", parent_span_id="parent001") - self._run(writer.write_batch([span])) + await writer.write_batch([span]) cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] assert any("CHILD_OF" in c for c in cypher_calls) - def test_no_child_of_when_no_parent(self): + async def test_no_child_of_when_no_parent(self): """CHILD_OF is NOT issued when parent_span_id is None.""" from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter db_manager, session = _make_db_manager() writer = AsyncOtelWriter(db_manager) span = _make_span(parent_span_id=None) - self._run(writer.write_batch([span])) + await writer.write_batch([span]) cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] assert not any("CHILD_OF" in c for c in cypher_calls) - def test_write_batch_creates_correlates_to_for_fqn(self): + async def test_write_batch_creates_correlates_to_for_fqn(self): """CORRELATES_TO relationship is attempted when fqn is set.""" from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter db_manager, session = _make_db_manager() writer = AsyncOtelWriter(db_manager) span = _make_span(fqn="App\\Controllers::index") - self._run(writer.write_batch([span])) + await writer.write_batch([span]) cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] assert any("CORRELATES_TO" in c for c in cypher_calls) - def test_no_correlates_to_when_no_fqn(self): + async def test_no_correlates_to_when_no_fqn(self): """CORRELATES_TO is NOT issued when fqn is None (no code context).""" from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter db_manager, session = _make_db_manager() writer = AsyncOtelWriter(db_manager) span = _make_span(fqn=None) - self._run(writer.write_batch([span])) + await writer.write_batch([span]) cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] assert not any("CORRELATES_TO" in c for c in cypher_calls) - def test_cross_service_span_creates_calls_service(self): + async def test_cross_service_span_creates_calls_service(self): """CALLS_SERVICE is created for CLIENT spans with peer_service set.""" from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter db_manager, session = _make_db_manager() writer = AsyncOtelWriter(db_manager) span = _make_span(cross_service=True, peer_service="payment-service") - self._run(writer.write_batch([span])) + await writer.write_batch([span]) cypher_calls = [str(c.args[0]) for c in session.run.call_args_list] assert any("CALLS_SERVICE" in c for c in cypher_calls) - def test_db_failure_routes_to_dlq(self): + async def test_db_failure_routes_to_dlq(self): """When the database raises, spans are moved to the dead-letter queue.""" from cgc_plugin_otel.neo4j_writer import AsyncOtelWriter db_manager = MagicMock() @@ -182,6 +180,6 @@ def test_db_failure_routes_to_dlq(self): writer = AsyncOtelWriter(db_manager) span = _make_span() - self._run(writer.write_batch([span])) + await writer.write_batch([span]) assert not writer._dlq.empty() diff --git a/tests/unit/plugin/test_otel_processor.py b/tests/unit/plugin/test_otel_processor.py index d7de53bf..e720a3f3 100644 --- a/tests/unit/plugin/test_otel_processor.py +++ b/tests/unit/plugin/test_otel_processor.py @@ -5,11 +5,11 @@ Tests MUST FAIL before T020 (span_processor.py) is implemented. """ import pytest -import sys -import os -# Allow import even when cgc-plugin-otel is not installed -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../../plugins/cgc-plugin-otel/src")) +cgc_plugin_otel = pytest.importorskip( + "cgc_plugin_otel", + reason="cgc-plugin-otel is not installed; skipping otel processor unit tests", +) # --------------------------------------------------------------------------- diff --git a/tests/unit/plugin/test_xdebug_parser.py b/tests/unit/plugin/test_xdebug_parser.py index 486475e8..966971a1 100644 --- a/tests/unit/plugin/test_xdebug_parser.py +++ b/tests/unit/plugin/test_xdebug_parser.py @@ -5,10 +5,11 @@ Tests MUST FAIL before T030 (dbgp_server.py) is implemented. """ import pytest -import sys -import os -sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../../plugins/cgc-plugin-xdebug/src")) +cgc_plugin_xdebug = pytest.importorskip( + "cgc_plugin_xdebug", + reason="cgc-plugin-xdebug is not installed; skipping xdebug parser unit tests", +) _SAMPLE_STACK_XML = """\ From 09615d4f8d4cf77a9abbd36c087d01b29325c3f6 Mon Sep 17 00:00:00 2001 From: Brian Gebel Date: Sat, 14 Mar 2026 21:47:16 -0700 Subject: [PATCH 4/7] fix(ci): replace manylinux2014 image with manylinux_2_39 for glibc 2.39+ compatibility falkordblite requires glibc 2.39+, which is not available in the manylinux2014_x86_64 image (glibc 2.17). Switch the Linux PyInstaller docker build to quay.io/pypa/manylinux_2_39_x86_64. Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3481810c..ac5530c8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -53,7 +53,7 @@ jobs: run: | docker run --rm \ -v ${{ github.workspace }}:/src \ - quay.io/pypa/manylinux2014_x86_64 \ + quay.io/pypa/manylinux_2_39_x86_64 \ /bin/bash -c " set -e cd /src From 56a45b9d5656960272b73d49312089b84ec01c29 Mon Sep 17 00:00:00 2001 From: Brian Gebel Date: Sat, 14 Mar 2026 21:51:03 -0700 Subject: [PATCH 5/7] fix(plugins): point entry points to package modules and add re-exports to __init__.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit All four plugin pyproject.toml files declared entry points with a module:attribute suffix (e.g. cgc_plugin_stub.cli:get_plugin_commands), causing ep.load() to return a function rather than the module. The PluginRegistry calls ep.load() and reads PLUGIN_METADATA, get_plugin_commands, get_mcp_tools, and get_mcp_handlers off the returned object as module attributes — so loading always failed. Fix: remove the colon-attribute suffix so each entry point resolves to the package module (e.g. cgc_plugin_stub). Add re-exports of get_plugin_commands, get_mcp_tools, and get_mcp_handlers from the cli/mcp_tools submodules into each plugin's __init__.py so the registry finds them on the loaded module. Co-Authored-By: Claude Sonnet 4.6 --- plugins/cgc-plugin-memory/pyproject.toml | 4 ++-- plugins/cgc-plugin-memory/src/cgc_plugin_memory/__init__.py | 3 +++ plugins/cgc-plugin-otel/pyproject.toml | 4 ++-- plugins/cgc-plugin-otel/src/cgc_plugin_otel/__init__.py | 3 +++ plugins/cgc-plugin-stub/pyproject.toml | 4 ++-- plugins/cgc-plugin-stub/src/cgc_plugin_stub/__init__.py | 5 +++++ plugins/cgc-plugin-xdebug/pyproject.toml | 4 ++-- plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/__init__.py | 3 +++ 8 files changed, 22 insertions(+), 8 deletions(-) diff --git a/plugins/cgc-plugin-memory/pyproject.toml b/plugins/cgc-plugin-memory/pyproject.toml index c31b6d59..5fe8f691 100644 --- a/plugins/cgc-plugin-memory/pyproject.toml +++ b/plugins/cgc-plugin-memory/pyproject.toml @@ -25,10 +25,10 @@ dev = [ ] [project.entry-points."cgc_cli_plugins"] -memory = "cgc_plugin_memory.cli:get_plugin_commands" +memory = "cgc_plugin_memory" [project.entry-points."cgc_mcp_plugins"] -memory = "cgc_plugin_memory.mcp_tools:get_mcp_tools" +memory = "cgc_plugin_memory" [tool.setuptools.packages.find] where = ["src"] diff --git a/plugins/cgc-plugin-memory/src/cgc_plugin_memory/__init__.py b/plugins/cgc-plugin-memory/src/cgc_plugin_memory/__init__.py index bd9db99a..fdf68af8 100644 --- a/plugins/cgc-plugin-memory/src/cgc_plugin_memory/__init__.py +++ b/plugins/cgc-plugin-memory/src/cgc_plugin_memory/__init__.py @@ -1,5 +1,8 @@ """Memory plugin for CodeGraphContext — stores and searches project knowledge in the graph.""" +from cgc_plugin_memory.cli import get_plugin_commands +from cgc_plugin_memory.mcp_tools import get_mcp_handlers, get_mcp_tools + PLUGIN_METADATA = { "name": "cgc-plugin-memory", "version": "0.1.0", diff --git a/plugins/cgc-plugin-otel/pyproject.toml b/plugins/cgc-plugin-otel/pyproject.toml index ac3b07f6..6adb46ac 100644 --- a/plugins/cgc-plugin-otel/pyproject.toml +++ b/plugins/cgc-plugin-otel/pyproject.toml @@ -30,10 +30,10 @@ dev = [ ] [project.entry-points."cgc_cli_plugins"] -otel = "cgc_plugin_otel.cli:get_plugin_commands" +otel = "cgc_plugin_otel" [project.entry-points."cgc_mcp_plugins"] -otel = "cgc_plugin_otel.mcp_tools:get_mcp_tools" +otel = "cgc_plugin_otel" [tool.setuptools.packages.find] where = ["src"] diff --git a/plugins/cgc-plugin-otel/src/cgc_plugin_otel/__init__.py b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/__init__.py index 66bbe9e6..50bd2309 100644 --- a/plugins/cgc-plugin-otel/src/cgc_plugin_otel/__init__.py +++ b/plugins/cgc-plugin-otel/src/cgc_plugin_otel/__init__.py @@ -1,5 +1,8 @@ """OTEL plugin for CodeGraphContext — receives OpenTelemetry spans and writes them to the graph.""" +from cgc_plugin_otel.cli import get_plugin_commands +from cgc_plugin_otel.mcp_tools import get_mcp_handlers, get_mcp_tools + PLUGIN_METADATA = { "name": "cgc-plugin-otel", "version": "0.1.0", diff --git a/plugins/cgc-plugin-stub/pyproject.toml b/plugins/cgc-plugin-stub/pyproject.toml index 71d73e2d..c388f8d2 100644 --- a/plugins/cgc-plugin-stub/pyproject.toml +++ b/plugins/cgc-plugin-stub/pyproject.toml @@ -21,10 +21,10 @@ dev = [ ] [project.entry-points."cgc_cli_plugins"] -stub = "cgc_plugin_stub.cli:get_plugin_commands" +stub = "cgc_plugin_stub" [project.entry-points."cgc_mcp_plugins"] -stub = "cgc_plugin_stub.mcp_tools:get_mcp_tools" +stub = "cgc_plugin_stub" [tool.setuptools.packages.find] where = ["src"] diff --git a/plugins/cgc-plugin-stub/src/cgc_plugin_stub/__init__.py b/plugins/cgc-plugin-stub/src/cgc_plugin_stub/__init__.py index d4185be7..7351c00d 100644 --- a/plugins/cgc-plugin-stub/src/cgc_plugin_stub/__init__.py +++ b/plugins/cgc-plugin-stub/src/cgc_plugin_stub/__init__.py @@ -1,8 +1,13 @@ """Stub plugin for testing the CGC plugin system.""" +from cgc_plugin_stub.cli import get_plugin_commands +from cgc_plugin_stub.mcp_tools import get_mcp_handlers, get_mcp_tools + PLUGIN_METADATA = { "name": "cgc-plugin-stub", "version": "0.1.0", "cgc_version_constraint": ">=0.1.0", "description": "Minimal stub plugin for testing CGC plugin discovery and loading.", } + +__all__ = ["PLUGIN_METADATA", "get_plugin_commands", "get_mcp_tools", "get_mcp_handlers"] diff --git a/plugins/cgc-plugin-xdebug/pyproject.toml b/plugins/cgc-plugin-xdebug/pyproject.toml index c338c412..464cb093 100644 --- a/plugins/cgc-plugin-xdebug/pyproject.toml +++ b/plugins/cgc-plugin-xdebug/pyproject.toml @@ -25,10 +25,10 @@ dev = [ ] [project.entry-points."cgc_cli_plugins"] -xdebug = "cgc_plugin_xdebug.cli:get_plugin_commands" +xdebug = "cgc_plugin_xdebug" [project.entry-points."cgc_mcp_plugins"] -xdebug = "cgc_plugin_xdebug.mcp_tools:get_mcp_tools" +xdebug = "cgc_plugin_xdebug" [tool.setuptools.packages.find] where = ["src"] diff --git a/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/__init__.py b/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/__init__.py index f7d7b1e7..6b4556ae 100644 --- a/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/__init__.py +++ b/plugins/cgc-plugin-xdebug/src/cgc_plugin_xdebug/__init__.py @@ -4,6 +4,9 @@ It must be explicitly enabled via CGC_PLUGIN_XDEBUG_ENABLED=true. """ +from cgc_plugin_xdebug.cli import get_plugin_commands +from cgc_plugin_xdebug.mcp_tools import get_mcp_handlers, get_mcp_tools + PLUGIN_METADATA = { "name": "cgc-plugin-xdebug", "version": "0.1.0", From e2615f1d215e0cf99773065419bda48acf28106b Mon Sep 17 00:00:00 2001 From: Brian Gebel Date: Sat, 14 Mar 2026 22:20:58 -0700 Subject: [PATCH 6/7] fix(ci): switch Linux build to native ubuntu runner to satisfy falkordblite glibc 2.39 requirement Co-Authored-By: Claude Sonnet 4.6 --- .github/workflows/build.yml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ac5530c8..6953f454 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -15,6 +15,7 @@ jobs: name: Build on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: [ubuntu-latest, windows-latest, macos-latest] include: @@ -48,20 +49,17 @@ jobs: run: | pyinstaller cgc.spec --clean - - name: Build with PyInstaller (Linux - Manylinux) + - name: Install dependencies (Linux) if: runner.os == 'Linux' run: | - docker run --rm \ - -v ${{ github.workspace }}:/src \ - quay.io/pypa/manylinux_2_39_x86_64 \ - /bin/bash -c " - set -e - cd /src - /opt/python/cp312-cp312/bin/python -m pip install --upgrade pip - /opt/python/cp312-cp312/bin/pip install . pyinstaller - /opt/python/cp312-cp312/bin/pyinstaller cgc.spec --clean - " - sudo chown -R $USER:$USER dist build + python -m pip install --upgrade pip + pip install . + pip install pyinstaller + + - name: Build with PyInstaller (Linux) + if: runner.os == 'Linux' + run: | + pyinstaller cgc.spec --clean - name: Rename artifact (Linux/Mac) if: runner.os != 'Windows' From 1cb69514e3a152bc82128d4bd08a215f957f7964 Mon Sep 17 00:00:00 2001 From: Brian Gebel Date: Sat, 14 Mar 2026 22:25:17 -0700 Subject: [PATCH 7/7] fix(spec): explicitly bundle falkordblite.libs and dummy extension for Linux frozen binary MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Linux manylinux_2_39_x86_64 falkordblite wheel (produced by auditwheel) installs vendored shared libraries into a `falkordblite.libs/` directory (libcrypto, libssl, libgomp) that is not a Python package. collect_all() and collect_dynamic_libs() only walk the importable top-level packages listed in top_level.txt ('dummy', 'redislite') and therefore silently miss falkordblite.libs/, causing runtime linker failures inside the PyInstaller one-file executable. Changes: - Add explicit search_paths scan for falkordblite.libs/*.so* and register each file as a binary with destination 'falkordblite.libs' - Collect dummy.cpython-*.so (auditwheel sentinel extension) from site-packages roots and register it at the bundle root - Add 'dummy' to hidden_imports (non-Windows) to cover the importable top-level package declared in falkordblite's top_level.txt pyproject.toml dependency marker (sys_platform != 'win32' and python_version >= '3.12') is correct — no change needed there. Co-Authored-By: Claude Sonnet 4.6 --- cgc.spec | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/cgc.spec b/cgc.spec index d01deff6..a83a93a3 100644 --- a/cgc.spec +++ b/cgc.spec @@ -259,6 +259,44 @@ if not is_win: except Exception as e: print(f"Warning: collect_all failed for {pkg}: {e}") +# ── falkordblite: explicit auditwheel-vendored shared library collection ────── +# The Linux manylinux wheel ships shared libraries in a `falkordblite.libs/` +# directory (placed by auditwheel alongside the importable packages). This +# directory is NOT a Python package (no __init__.py), so collect_all/ +# collect_dynamic_libs operating on top-level package names ('dummy', +# 'redislite') will not find it. We must explicitly scan for it and register +# every .so* in it as a binary so the dynamic linker can resolve libcrypto, +# libssl, and libgomp at runtime inside the frozen one-file executable. +# +# On macOS the equivalent vendored dylibs live in redislite/.dylibs/ and are +# already captured by collect_all('falkordblite') above. On Windows the +# package is not installed (sys_platform != 'win32' marker), so this block +# is also guarded. +if not is_win: + for _sp in search_paths: + _fdb_libs = _sp / 'falkordblite.libs' + if _fdb_libs.exists(): + for _lib in _fdb_libs.iterdir(): + if _lib.is_file() and not _lib.suffix == '.py': + print(f"Bundling falkordblite.libs: {_lib}") + binaries.append((str(_lib), 'falkordblite.libs')) + break # found; no need to check further paths + +# falkordblite ships a top-level 'dummy' C extension (dummy.cpython-*.so). +# It is not an application import but must travel with the bundle as a +# sentinel that pip/auditwheel attaches native build metadata to. +# Collect it explicitly so PyInstaller does not silently drop it. +if not is_win: + # 'dummy' may resolve to the stdlib dummy module; guard by only picking up + # the .so file that lives directly in a site-packages root (where auditwheel + # places it) rather than in any sub-package directory. + for _sp in search_paths: + for _dummy_so in _sp.glob('dummy.cpython-*.so'): + if _dummy_so.is_file(): + print(f"Bundling falkordblite dummy extension: {_dummy_so}") + binaries.append((str(_dummy_so), '.')) + break + # stdlibs: dynamically imports py3.py, py312.py, etc. via importlib stdlibs_dir = find_pkg_dir('stdlibs') if stdlibs_dir: @@ -283,6 +321,10 @@ if tslp_dir: # Add redislite submodules to hidden imports hidden_imports += collect_submodules('redislite') hidden_imports += collect_submodules('falkordb') +# falkordblite's top_level.txt declares 'dummy' and 'redislite'. +# 'redislite' is covered above; add 'dummy' explicitly. +if not is_win: + hidden_imports.append('dummy') # Add platform-specific watchers if is_win: