diff --git a/.coveragerc b/.coveragerc
index 122ffd55..e951249a 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -17,6 +17,8 @@ omit =
*/website_profiling/llm/providers/*
*/website_profiling/llm/*
*/website_profiling/llm_config.py
+ */website_profiling/llm_client_http.py
+ */website_profiling/commands/chat_cmd.py
*/website_profiling/cli.py
*/website_profiling/commands/enrich_cmd.py
# FastAPI server — tested via integration tests, not unit tests
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4e977f19..d3db1bdb 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -133,3 +133,23 @@ jobs:
dotnet-version: '10.0.x'
- name: Test Data service
run: dotnet test services/Data/Data.slnx
+
+ ai:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-dotnet@v4
+ with:
+ dotnet-version: '10.0.x'
+ - name: Test AiService
+ run: dotnet test services/AiService/AiService.slnx
+
+ integrations:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-dotnet@v4
+ with:
+ dotnet-version: '10.0.x'
+ - name: Test IntegrationsService
+ run: dotnet test services/IntegrationsService/IntegrationsService.slnx
diff --git a/AGENT.md b/AGENT.md
index fcd3c030..d972c89d 100644
--- a/AGENT.md
+++ b/AGENT.md
@@ -4,19 +4,21 @@ Developer reference for agents and contributors. User-facing overview: [README.m
**What it is:** `python -m src` from repo root (`src/__main__.py` -> package **`website_profiling`**). Config: stored in **PostgreSQL** (`pipeline_config` table, `key/value/is_unknown/updated_at`). A shadow **`pipeline-config.txt`** is auto-written to `DATA_DIR` on every Save/Run. CLI loads DB first (`DATABASE_URL`), then shadow file; `--config` overrides with a file. Reference keys: `input.txt.example` and `pipeline-config.example.txt` (not auto-loaded).
-**LLM / AI:** Settings live in **`llm_config`** table in PostgreSQL. Providers: OpenAI, Google Gemini, Anthropic, Groq, Ollama (`web/src/lib/llmConfigSchema.ts`). Configure only via web UI **AI** tab (`GET/PUT /api/llm-config`, localhost). Never in `pipeline-config.txt` or `--config`.
+**LLM / AI:** Settings live in **`llm_config`** (and related tables) in PostgreSQL. Providers: OpenAI, Google Gemini, Anthropic, Groq, Ollama (`web/src/lib/llmConfigSchema.ts`). **Browser writes** for API keys and LLM toggles go **BFF → AiService** (`PUT /api/secrets`, `PUT /api/llm-config`). Configure via **Secrets** (`/secrets`) and **Run audit → AI settings**. Never in `pipeline-config.txt` or `--config`. Worker/CLI calls AiService via `llm_client_http.py` (`AI_SERVICE_URL`, default `:8092`).
-**Frontend:** **`web/`** (Vite + React SPA) — browser calls **`services/Bff/`** for all `/api/*`; BFF proxies to FastAPI and FileService.
+**Frontend:** **`web/`** (Vite + React SPA) — browser calls **`services/Bff/`** for all `/api/*`; BFF proxies to FastAPI, AiService, Data, and FileService.
**Key paths**
-- `src/website_profiling/` -- `cli.py`, `config.py`, `crawl/`, `db/storage.py`, `lighthouse/`, `reporting/`, `analysis/`, `llm/`, `tools/`
-- `services/Bff/` -- .NET BFF (auth, CORS, `/api/*` proxy)
+- `src/website_profiling/` -- `cli.py`, `config.py`, `crawl/`, `db/storage.py`, `lighthouse/`, `reporting/`, `analysis/`, `llm_client_http.py`, `tools/`
+- `services/Bff/` -- .NET BFF (auth, CORS, `/api/*` proxy to FastAPI + AiService + Data + FileService)
+- `services/AiService/` -- .NET AI (chat, secrets, LLM config, MCP, enrichment; port 8092)
+- `services/Data/` -- .NET read service (report payloads, portfolio, issue status; port 8091)
- `services/FileService/` -- .NET PDF + Excel workbook export (HTTP-only; see [README](services/FileService/README.md))
- `web/src/` -- React SPA (`AppRoutes.tsx`, `views/`, `components/`); pipeline UI: `PipelineRunnerFab`, `PipelineContext`
- `alembic/` -- schema migrations
-**Local dev:** `./local-run` (Postgres in Docker `wp-pg`, FileService on `:8080`, FastAPI on `:8001`, BFF on `:8090`, Vite on `:3000`; default `DATABASE_URL`: `postgres://postgres:dev@127.0.0.1:5432/website_profiling`). See `scripts/local-run.sh`. **Local tests:** `./local-test` runs **three** Python coverage gates (core 100%, reporting 100%, tools 100%) plus web checks — mirrors CI **python** and **web** jobs; Docker CI is separate (see `.github/workflows/ci.yml`). `./local-test browser` for `@pytest.mark.browser` integration tests — see `scripts/local-test.sh`. Mocked browser unit tests: `tests/test_browser_fetcher_unit.py`.
+**Local dev:** `./local-run` (Postgres in Docker `wp-pg`, FileService `:8080`, Data `:8091`, AiService `:8092`, FastAPI `:8001`, BFF `:8090`, Vite `:3000`; default `DATABASE_URL`: `postgres://postgres:dev@127.0.0.1:5432/website_profiling`). See `scripts/local-run.sh`. **Local tests:** `./local-test` runs **three** Python coverage gates (core 100%, reporting 100%, tools 100%) plus web and .NET checks — mirrors CI; Docker CI is separate (see `.github/workflows/ci.yml`). `./local-test browser` for `@pytest.mark.browser` integration tests — see `scripts/local-test.sh`. Mocked browser unit tests: `tests/test_browser_fetcher_unit.py`.
**JavaScript crawl (optional):** Config keys `crawl_render_mode` (`static` | `javascript` | `auto`) and `crawl_js_*` in pipeline config / `pipelineConfigSchema.ts`. JS/auto crawls can capture browser console errors and uncaught exceptions (`crawl_js_capture_console`, stored under `page_analysis.browser`). **Auto mode** uses static-first fetch, pre-parse SPA heuristics (`needs_js_render`), then post-parse low-outlink fallback (`needs_js_render_after_parse`) in `crawler.py`. **Preflight:** `GET /api/crawl/browser-status` (localhost) spawns Python `browser_status()`; Run audit settings/run validation calls it when render mode is `javascript` or `auto`. Browser deps: Playwright from `requirements.txt` (installed by `./local-run setup` and `./local-test`). Runtime needs Chromium on `PATH` or `CHROME_PATH` (Docker sets `CHROME_PATH=/usr/bin/chromium`). Integration tests: `@pytest.mark.browser` — excluded by default in `pytest.ini`; Docker CI runs `tests/test_crawl_fetchers.py` and `tests/test_crawler_browser_e2e.py -m browser`; locally `./local-test browser`.
@@ -25,15 +27,15 @@ Developer reference for agents and contributors. User-facing overview: [README.m
- Run audit (CLI): `python -m src` — reads config from PostgreSQL (`pipeline_config`); shadow `DATA_DIR/pipeline-config.txt` if table empty. CLI override: `python -m src --config path`
- Optional step: `crawl` | `report` | `plot` | `lighthouse` | `keywords` | `warnings` | `enrich` | `google` | `chat`
- **`preserve_crawl_history`** (default true): append crawls; `false` truncates crawl tables but restores `report_payload`, Lighthouse, `google_data`, `keyword_data`, `keyword_history`, `keyword_suggest_cache`, and `crawl_runs`
-- **`DATABASE_URL`** env: PostgreSQL connection string (required). **`DATA_DIR`**: secrets + shadow config (Docker: `/data`).
+- **`DATABASE_URL`** env: PostgreSQL connection string (required). **`DATA_DIR`**: shadow pipeline config and local artifacts (Docker: `/data`); API keys live in Postgres via AiService.
- **Pipeline storage** (crawl, edges, nodes, report payload, Lighthouse, keywords, warnings) lives in **PostgreSQL only**. Deliverables use the Export view, `GET /api/report/export`, or MCP `export_*` tools — not files written by the main pipeline step.
- **Pool tuning:** `DB_POOL_MIN` / `DB_POOL_MAX` (Python). Bulk crawl writes via `executemany`; optional **`crawl_stream_to_db`** streams rows during fetch. Per-URL raw HTML: `crawl_page_html` table (migration `015`); API `GET/POST /api/crawl/page-html`.
-- **Browser API (BFF):** All `/api/*` routes are served by `services/Bff/` (proxied to FastAPI / FileService). Notable: `/api/report/*`, `/api/run`, `/api/jobs/*`, `/api/pipeline-config`, `/api/llm-config`, `/api/chat` (SSE), `/api/integrations/google/*` (OAuth callback on BFF origin). `PipelineRunnerFab` saves pipeline + LLM state before each run. OpenAPI: `web/openapi.json`; BFF client: `services/Bff/src/Bff.Application/Generated/`.
-- **MCP:** `python -m website_profiling.mcp` (stdio) or `python -m website_profiling.mcp.http` (remote Streamable HTTP). Configure at **`/mcp`** in the web UI. See `docs/MCP.md`.
+- **Browser API (BFF):** All `/api/*` routes are served by `services/Bff/`. **FastAPI:** `/api/run`, `/api/jobs/*`, `/api/pipeline-config`, crawl, integrations (OAuth reads), properties, content drafts, etc. **AiService:** `/api/chat` (SSE), `/api/llm-config`, `/api/secrets`, `/api/ollama/status`, `/api/issues/fix-suggestion`, `/api/issues/action-plan`, `/api/dashboards/ai-generate`, `/api/content/analyze`, `/api/content/wizard`, `/api/links/page-coach`, `/api/mcp-tools`, `/api/report/audit-tool`. **Data:** report payload reads, portfolio, issue status, saved filters (see `DATA_ROUTES`). **FileService:** PDF/workbook export. `PipelineRunnerFab` saves pipeline config (FastAPI) and LLM state (`PUT /api/llm-config` → AiService) before each run. OpenAPI: `web/openapi.json` (FastAPI routes only — AiService routes are not in this spec); BFF client: `services/Bff/src/Bff.Application/Generated/`.
+- **MCP:** AiService (.NET) — stdio host or HTTP at `/mcp` when `WP_MCP_HTTP=1` on `:8092`. Configure at **`/mcp`** in the web UI. See `docs/MCP.md` and [services/AiService/README.md](services/AiService/README.md).
- **AI Chat UI:** `/chat` — property-scoped chat with saved sessions (`chat_sessions`, `chat_messages`; migration `012_chat_sessions`).
- **Job store:** PostgreSQL `pipeline_jobs` (FastAPI); live job status via `/api/jobs/*` through the BFF.
- **Schema head:** `015_crawl_page_html` (recent: `013` link_edges/discovery, `014` job log truncation, `015` per-URL HTML storage).
-- **Docker:** Root `Dockerfile` (Python backend); `web/Dockerfile` (Vite SPA + nginx); `docker-compose.yml` (postgres + fastapi + worker + bff + web + FileService); **`docker-compose.prod.yml`** (production + optional MCP on `:8000`); **`docker-compose.pull.yml`** for pre-built images (`BACKEND_IMAGE`, `WEB_IMAGE`); **`LIGHTHOUSE_CHROME_FLAGS`**
+- **Docker:** Root `Dockerfile` (Python backend); `web/Dockerfile` (Vite SPA + nginx); `docker-compose.yml` (postgres + fastapi + worker + ai + data + bff + web + FileService); **`docker-compose.prod.yml`** (production + optional MCP profile mapping host `:8000` → AiService `:8092`); **`docker-compose.pull.yml`** for pre-built images (`BACKEND_IMAGE`, `WEB_IMAGE`); **`LIGHTHOUSE_CHROME_FLAGS`**
**Where to edit**
@@ -44,12 +46,13 @@ Developer reference for agents and contributors. User-facing overview: [README.m
| PDF / workbook export | `services/FileService/` (rendering); BFF routes `/api/report/export` and `/api/report/export-workbook` to FileService |
| DB schema | `alembic/versions/` |
| Local analysis | `analysis/local.py`, `requirements.txt` |
-| AI insights (LLM) | `llm/enrich.py`, `llm/agent.py`, `llm_config.py`, `requirements.txt` |
-| Audit query tools (MCP + chat) | `tools/audit_tools/`, `mcp/server.py`, `mcp/http_server.py`, `commands/chat_cmd.py` |
+| AI insights (LLM) | `services/AiService/` (browser-facing), `llm_client_http.py` (worker/CLI), `llm_config.py` |
+| Audit query tools (MCP + chat) | `services/AiService/src/AiService.Tools/`, `services/AiService/src/AiService.Mcp/`, `tools/audit_tools/`, `commands/chat_cmd.py` |
| Agent readiness checks | `tools/audit_tools/geo/agent_readiness.py`, `tools/audit_tools/_aeo_helpers.py` |
| Config / CLI | `config.py` (`load_config`, `load_config_from_db`), `cli.py`, `input.txt.example` |
| UI pipeline schema | `web/src/lib/pipelineConfigSchema.ts` |
| UI LLM schema | `web/src/lib/llmConfigSchema.ts` |
+| UI secrets schema | `web/src/lib/secretsConfigSchema.ts`, `web/src/hooks/useSecrets.ts` |
| Browser API client | `web/src/lib/publicBase.ts` (`apiUrl`, `apiFetch`, `VITE_BFF_BASE_URL`) |
| D3 charts (custom / compare / overview) | `web/src/components/charts/d3/`, `web/src/lib/viz/` |
| Chart.js charts (standard bar/line/doughnut) | `web/src/utils/chartJsDefaults.ts`, `react-chartjs-2` in views under `web/src/views/`, `web/src/components/searchPerformance/`, `web/src/components/traffic/` |
diff --git a/AGENTS.md b/AGENTS.md
index 797c472b..d7e40319 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -4,15 +4,17 @@
This file is the canonical entry point for agents. For full detail see [AGENT.md](AGENT.md).
-**What it is:** Self-hosted SEO crawl and technical audit platform — `python -m src` from repo root. Stack: Python (crawl + analysis + MCP + FastAPI), Vite + React SPA (web UI), .NET BFF (browser API), .NET Data (report reads), PostgreSQL.
+**What it is:** Self-hosted SEO crawl and technical audit platform — `python -m src` from repo root. Stack: Python (crawl + analysis + FastAPI), Vite + React SPA (web UI), .NET BFF (browser API), .NET Data (reads), .NET AiService (AI/LLM/MCP), .NET IntegrationsService (Google/Bing I/O), PostgreSQL.
**Key paths**
- `src/website_profiling/` — core Python package
- - `cli.py`, `config.py`, `api/`, `worker/`, `crawl/`, `db/`, `reporting/`, `analysis/`, `llm/`, `tools/`
+ - `cli.py`, `config.py`, `api/`, `worker/`, `crawl/`, `db/`, `reporting/`, `analysis/`, `llm_client_http.py`, `tools/`
- `web/` — Vite + React SPA (static nginx in prod); browser calls `services/Bff/` for all `/api/*`
-- `services/Bff/` — .NET BFF (auth, CORS, proxy to FastAPI + Data + FileService)
+- `services/Bff/` — .NET BFF (auth, CORS, proxy to FastAPI + IntegrationsService + Data + AiService + FileService)
- `services/Data/` — .NET read service (report payloads, portfolio, issue status, filters; port 8091)
+- `services/AiService/` — .NET AI service (Microsoft.Extensions.AI, chat, enrichment, MCP, **secrets/llm-config writes**; port 8092). See [services/AiService/README.md](services/AiService/README.md)
+- `services/IntegrationsService/` — .NET Google/Bing integrations (GSC/GA4 fetch, OAuth, page-live, keyword reads; port 8093). See [services/IntegrationsService/README.md](services/IntegrationsService/README.md)
- `services/FileService/` — .NET PDF + Excel workbook export (port 8080). HTTP-only via `REPORT_API_URL`; no Postgres. Profiles: `executive|standard|full|premium`. Details: [services/FileService/README.md](services/FileService/README.md). Env: `FILE_SERVICE_URL` (MCP), `REPORT_API_URL` (FileService).
- `alembic/` — DB migrations
- `docs/` — documentation index
@@ -21,13 +23,15 @@ This file is the canonical entry point for agents. For full detail see [AGENT.md
**Run / dev**
```bash
-./local-run # Start Postgres + FileService + Data + worker + FastAPI + BFF + Vite dev server
+./local-run # Start Postgres + FileService + Data + AiService + IntegrationsService + worker + FastAPI + BFF + Vite
./local-test # Python + web + .NET tests (CI parity)
python -m src # Run audit pipeline
-python -m website_profiling.mcp # Start MCP server (stdio)
+# MCP: AiService stdio/HTTP — see services/AiService/README.md and docs/MCP.md
```
-**MCP:** 340 read-only audit tools via Model Context Protocol. See [docs/MCP.md](docs/MCP.md).
+**MCP:** 369 read-only audit tools via Model Context Protocol (AiService). See [docs/MCP.md](docs/MCP.md).
+
+**Secrets / credentials:** Browser writes go BFF → AiService only (`PUT /api/secrets`, `PUT /api/llm-config`). Python FastAPI keeps `pipeline-config` and read-only integration routes; worker/crawl reads `llm_config` / `google_app_settings` from Postgres at runtime.
**Edit targets**
diff --git a/README.md b/README.md
index 070bfd7f..c465f7a1 100644
--- a/README.md
+++ b/README.md
@@ -41,7 +41,7 @@
# Site Audit
-**Developer-friendly SEO audit platform** — open-source crawl and technical audit tooling built with **React, Python, PostgreSQL, and .NET**. The stack is split into focused services: a **Python FastAPI** backend (crawl, pipeline, chat, integrations), a **.NET BFF** as the browser-facing API gateway, a **.NET Data** read service (report payloads, portfolio, issue status), and a **.NET FileService** for PDF/Excel export.
+**Developer-friendly SEO audit platform** — open-source crawl and technical audit tooling built with **React, Python, PostgreSQL, and .NET**. The stack is split into focused services: **Python FastAPI** (crawl, pipeline, pipeline-config), **.NET IntegrationsService** (Google/Bing OAuth, GSC/GA4 fetch, keywords), **.NET AiService** (AI chat, LLM config, secrets, MCP), **.NET Data** (report reads, portfolio, issue status), **.NET FileService** (PDF/Excel export), and a **.NET BFF** as the single browser-facing API gateway.
## Overview
@@ -71,7 +71,7 @@ Site Audit is a **developer-friendly SEO audit** tool: self-hosted, transparent,
## SEO feedback loop
-Site Audit is built for a **continuous improve-and-verify cycle**, not one-off dashboard checks. Crawl your site, generate reports, expose audit data to AI agents in **Cursor, Claude Code, or Copilot** via [340 MCP tools](docs/MCP.md), fix issues in your repository, then **review** the next run to compare health scores and issue deltas.
+Site Audit is built for a **continuous improve-and-verify cycle**, not one-off dashboard checks. Crawl your site, generate reports, expose audit data to AI agents in **Cursor, Claude Code, or Copilot** via [369 MCP tools](docs/MCP.md), fix issues in your repository, then **review** the next run to compare health scores and issue deltas.
```text
Audit → Report → MCP → Fix → Review → (repeat)
@@ -87,7 +87,7 @@ Audit → Report → MCP → Fix → Review → (repeat)
|------|-------------|---------------|
| **Audit** | Crawl and score the site | Pipeline (`python -m src`), Lighthouse, on-page checks |
| **Report** | Export and prioritize fixes | PDF, Excel workbook, CSV, and JSON exports; issue board; fix roadmap |
-| **MCP** | Pull audit context into your IDE | `python -m website_profiling.mcp` — read-only tools for Cursor / Claude Desktop |
+| **MCP** | Pull audit context into your IDE | AiService MCP (stdio or HTTP on `:8092`) — see [docs/MCP.md](docs/MCP.md) |
| **Fix** | Ship changes in your codebase | Your PR workflow (MCP does not write to the site) |
| **Review** | Prove improvement | Compare runs, category deltas, GSC metric changes |
@@ -134,18 +134,27 @@ Site Audit focuses on **honest, self-hosted technical SEO**. It is not a drop-in
-Also included: **AI chat** over audit data (optional), **Content studio** (write & optimize with live SEO scoring), **340 MCP tools** (local stdio or remote Streamable HTTP), image SEO, GEO/AEO readiness, keyword explorer (GSC + on-site), backlinks (GSC Links import), compare runs, portfolio management for agencies, and the **agent-driven feedback loop** above.
+Also included: **AI chat** over audit data (optional), **Content studio** (write & optimize with live SEO scoring), **369 MCP tools** (AiService stdio or remote Streamable HTTP), image SEO, GEO/AEO readiness, keyword explorer (GSC + on-site), backlinks (GSC Links import), compare runs, portfolio management for agencies, and the **agent-driven feedback loop** above.
## Architecture
-```text
-Browser → web (:3000) → bff (:8090) → fastapi (:8001) crawl, pipeline, chat, integrations
- │ data (:8091) report reads, portfolio, issue status, filters
- │ files (:8080) PDF + Excel export
- worker background pipeline jobs (same Python image)
- postgres audit data store
+```mermaid
+flowchart TB
+ Browser([Browser]) --> Web["web :3000
React SPA"]
+ Web --> BFF["bff :8090
.NET API gateway"]
+
+ BFF --> FastAPI["fastapi :8001
Crawl · pipeline · pipeline-config"]
+ BFF --> Integrations["integrations :8093
Google/Bing OAuth · GSC/GA4 · keywords"]
+ BFF --> Ai["ai :8092
Chat · LLM config · secrets · MCP"]
+ BFF --> Data["data :8091
Report reads · portfolio · issue status"]
+ BFF --> Files["files :8080
PDF · Excel export"]
+
+ subgraph Background["Background"]
+ Worker["worker
Pipeline jobs (same Python image)"]
+ Postgres[("postgres
Audit data store")]
+ end
```
```
@@ -159,9 +168,8 @@ WebsiteProfiling/
│ ├── content_studio/ # Content writing + live SEO scoring
│ ├── lighthouse/ # Lighthouse runner
│ ├── integrations/ # Google Search Console, GA4, Bing, CrUX
-│ ├── llm/ # AI enrich + chat agent
-│ ├── tools/ # Exports, audit query tools, MCP helpers
-│ ├── mcp/ # MCP server (stdio + remote HTTP, domain bundles)
+│ ├── llm_client_http.py # Worker/CLI → AiService HTTP client (enrichment, page coach, …)
+│ ├── tools/ # Exports, audit query tools, audit-tool bridge
│ ├── db/ # PostgreSQL storage layer
│ ├── commands/ # CLI subcommands
│ ├── cli.py # Pipeline entrypoint
@@ -173,6 +181,8 @@ WebsiteProfiling/
│ ├── src/lib/ # Client helpers, BFF apiUrl/apiFetch
│ └── public/ # Static assets (logo, favicon)
├── services/Bff/ # .NET BFF — auth + /api/* proxy (port 8090)
+├── services/IntegrationsService/ # .NET Google/Bing integrations — OAuth, GSC/GA4, keywords (port 8093)
+├── services/AiService/ # .NET AI — chat, secrets, LLM config, MCP, enrichment (port 8092)
├── services/Data/ # .NET read service — report/portfolio/issue reads (port 8091)
├── services/FileService/ # .NET PDF + Excel workbook export (port 8080)
├── alembic/versions/ # PostgreSQL schema migrations
@@ -194,9 +204,11 @@ WebsiteProfiling/
| Path | Purpose |
| ------------------------------------- | ------------------------------------------------------------------------------ |
-| `src/website_profiling/` | Crawl, analyze, report, Lighthouse, integrations, AI — run via `python -m src` |
-| `src/website_profiling/api/` | FastAPI HTTP layer — pipeline, chat, integrations, crawl control |
-| `services/Bff/` | Browser API gateway — auth, CORS, routes `/api/*` to FastAPI, Data, FileService |
+| `src/website_profiling/` | Crawl, analyze, report, Lighthouse — run via `python -m src` |
+| `src/website_profiling/api/` | FastAPI — pipeline, crawl, pipeline-config (not secrets/chat/integrations) |
+| `services/Bff/` | Browser API gateway — auth, CORS, routes `/api/*` to FastAPI, IntegrationsService, AiService, Data, FileService |
+| `services/IntegrationsService/` | Google/Bing OAuth, GSC/GA4 fetch, keywords, page-live — see [services/IntegrationsService/README.md](services/IntegrationsService/README.md) |
+| `services/AiService/` | AI chat, secrets, LLM config, MCP, enrichment — see [services/AiService/README.md](services/AiService/README.md) |
| `services/Data/` | .NET read/mutation service for report payloads, portfolio, issue status, saved filters |
| `services/FileService/` | PDF and Excel workbook export — see [services/FileService/README.md](services/FileService/README.md) |
| `web/src/lib/publicBase.ts` | BFF base URL (`VITE_BFF_BASE_URL`) and `apiFetch` / `apiUrl` |
@@ -204,7 +216,7 @@ WebsiteProfiling/
| `alembic/versions/` | Database migrations — run `./local-run migrate` |
| `tests/` | Backend tests; `./local-test browser` for Playwright crawl integration |
| `docs/MCP.md` | MCP server setup for IDE and agent integrations |
-| `data/` | Local secrets and shadow `pipeline-config.txt` (gitignored) |
+| `data/` | Shadow `pipeline-config.txt` and local artifacts (gitignored; secrets live in Postgres) |
For layout details and common development patterns, see [AGENT.md](AGENT.md).
@@ -218,7 +230,7 @@ For layout details and common development patterns, see [AGENT.md](AGENT.md).
| **Docker** | Postgres container (local dev) and full-stack compose |
| **Python 3.12+** | Audit engine, FastAPI, pipeline worker, tests |
| **Node 20+** | Vite + React SPA |
-| **.NET SDK 10+** | BFF, Data, and FileService (required for `./local-run`; optional if you only use Docker) |
+| **.NET SDK 10+** | BFF, IntegrationsService, AiService, Data, and FileService (required for `./local-run`; optional if you only use Docker) |
### Docker
@@ -228,9 +240,9 @@ Build and run the full dev stack from source:
docker compose up --build
```
-Services: **postgres**, **fastapi** (`:8001`, internal), **worker**, **data** (`:8091`, internal), **bff** (`:8090`), **web** (`:3000`), **files** (`:8080`, internal).
+Services: **postgres**, **fastapi** (`:8001`, internal), **worker**, **integrations** (`:8093`, internal), **ai** (`:8092`, internal), **data** (`:8091`, internal), **bff** (`:8090`), **web** (`:3000`), **files** (`:8080`, internal).
-Open [http://localhost:3000/home](http://localhost:3000/home). The browser talks only to the **BFF** (`:8090`); the BFF proxies to FastAPI, the Data service (report reads and portfolio routes), and FileService (PDF/workbook export).
+Open [http://localhost:3000/home](http://localhost:3000/home). The browser talks only to the **BFF** (`:8090`); the BFF proxies to FastAPI (crawl/pipeline), IntegrationsService (Google/Bing), AiService (AI/secrets), Data (report reads), and FileService (PDF/workbook export).
Production deployment: `docker-compose.prod.yml` — set `POSTGRES_USER`, `POSTGRES_PASSWORD`, `AUTH_SECRET`, `BFF_ALLOWED_ORIGINS`, and `BFF_PUBLIC_URL`. Optional remote MCP: `docker compose -f docker-compose.prod.yml --profile mcp up`. Pre-built images: `docker-compose.pull.yml` (`BACKEND_IMAGE`, `WEB_IMAGE`).
@@ -245,7 +257,7 @@ Production deployment: `docker-compose.prod.yml` — set `POSTGRES_USER`, `POSTG
./local-prod # Same DB, Vite production build + preview (no hot reload)
```
-`./local-run` starts (in order): **FileService** `:8080`, **Data** `:8091`, **pipeline worker**, **FastAPI** `:8001`, **BFF** `:8090`, and **Vite** `:3000`. Use `localhost` (not `127.0.0.1`) for pipeline APIs so CORS and cookies match the BFF origin.
+`./local-run` starts (in order): **FileService** `:8080`, **Data** `:8091`, **AiService** `:8092` (MCP HTTP enabled), **IntegrationsService** `:8093`, **pipeline worker**, **FastAPI** `:8001`, **BFF** `:8090`, and **Vite** `:3000`. Use `localhost` (not `127.0.0.1`) for pipeline APIs so CORS and cookies match the BFF origin.
Default local `DATABASE_URL`: `postgres://postgres:dev@127.0.0.1:5432/website_profiling` (Docker Compose dev stack uses `profiling:profiling`).
@@ -265,16 +277,16 @@ Increase `PIPELINE_JOB_STALE_HOURS` for crawls that routinely exceed one hour.
### Testing
```bash
-./local-test # Full CI parity: Python + web + .NET (Data, Bff, FileService)
+./local-test # Full CI parity: Python + web + .NET (Data, AiService, Bff, FileService)
./local-test python # Backend: three 100% coverage gates + browser pytest + CLI smoke
./local-test browser # JS crawl integration tests (skips if Chromium unavailable)
./local-test web # Frontend: build, typecheck, lint, vitest
-./local-test dotnet # dotnet test Data + Bff + FileService + BFF OpenAPI drift gate
+./local-test dotnet # dotnet test Data + AiService + Bff + FileService + BFF OpenAPI drift gate
./local-test quick # Fast loop; requires DB already running (no coverage gate)
./local-test all --no-cov # Full run without pytest coverage gate
```
-CI runs separate jobs for **Python** (coverage gates), **web**, **Data**, **Bff**, **FileService**, and **Docker** (image build, browser pytest in container, compose smoke). See [.github/workflows/ci.yml](.github/workflows/ci.yml).
+CI runs separate jobs for **Python** (coverage gates), **web**, **Data**, **AiService**, **IntegrationsService**, **Bff**, **FileService**, and **Docker** (image build, browser pytest in container, compose smoke). See [.github/workflows/ci.yml](.github/workflows/ci.yml).
## Configuration
@@ -298,9 +310,13 @@ The overlay enriches keywords that have no Search Console impressions with `plan
In Audit settings, set **Crawl rendering** to `javascript` (always headless Chromium) or `auto` (static first, browser when SPA heuristics match). Requires Playwright from `requirements.txt` and Chromium on `PATH` or `CHROME_PATH` (included in Docker). The UI preflights via `GET /api/crawl/browser-status` before runs when JS or auto mode is selected.
+### Secrets and AI settings
+
+API keys, Google OAuth app credentials, and remote MCP tokens are saved on the **Secrets** page (`/secrets`) — browser writes go **BFF → AiService** (`PUT /api/secrets`). LLM provider/model toggles and risk settings use **`PUT /api/llm-config`** (also AiService). Audit pipeline keys remain on **`PUT /api/pipeline-config`** (FastAPI). The pipeline run FAB saves LLM state via `/api/llm-config` before each run.
+
### AI chat (optional)
-Ask questions about audit data at [http://localhost:3000/chat](http://localhost:3000/chat). Enable a provider under **Run audit → AI settings** (`llm_enabled`, provider, model). `./local-run setup` installs Python deps from `requirements.txt` (including `httpx`, OpenAI, Anthropic, and Groq SDKs; Gemini uses `httpx` via REST).
+Ask questions about audit data at [http://localhost:3000/chat](http://localhost:3000/chat). Enable a provider under **Run audit → AI settings** (`llm_enabled`, provider, model) or on **Secrets**. AiService handles chat, tool dispatch, and streaming; `./local-run` starts it on `:8092`.
| Provider | Notes |
@@ -311,7 +327,7 @@ Ask questions about audit data at [http://localhost:3000/chat](http://localhost:
| **Groq** | API key in AI settings or `GROQ_API_KEY`; official Groq Python SDK; native tool calling with streaming. Default model `openai/gpt-oss-120b`. |
-The agent uses the same **340 read-only audit tools** as the MCP server ([docs/MCP.md](docs/MCP.md)), with **dynamic routing** (~45 tools per turn). Responses stream over SSE (`POST /api/chat`). Sessions persist per property (`chat_sessions` / `chat_messages`).
+The agent uses the same **369 read-only audit tools** as the MCP server ([docs/MCP.md](docs/MCP.md)), with **dynamic routing** (~45 tools per turn). Responses stream over SSE (`POST /api/chat` via BFF → AiService). Sessions persist per property (`chat_sessions` / `chat_messages`).
**Read-only SQL chat tool (opt-in):** Set `CHAT_SQL_TOOL_ENABLED=true` to expose `get_sql_schema` and `run_sql_query` to the LLM. The agent can then answer arbitrary data questions by generating and executing a single read-only SELECT. Queries are validated by a four-layer guard (regex pre-filter → `sqlglot` AST + table allowlist → `BEGIN TRANSACTION READ ONLY` → optional least-privilege DB role); DELETE/UPDATE/INSERT/DDL and non-allowlisted tables are always blocked. In multi-property deployments, scope-binding CTEs are automatically injected to enforce tenant isolation. See [docs/OPS.md](docs/OPS.md#read-only-sql-chat-tool) for setup including the recommended `audit_readonly` Postgres role and optional RLS configuration.
@@ -337,6 +353,8 @@ Contributions are welcome. See [CONTRIBUTING.md](CONTRIBUTING.md) for setup and
| [docs/COMPANY_STANDARDS.md](docs/COMPANY_STANDARDS.md) | Data and security policy |
| [docs/MCP.md](docs/MCP.md) | MCP server setup |
| [docs/OPS.md](docs/OPS.md) | Scheduled audits, alerts, production ops |
+| [services/AiService/README.md](services/AiService/README.md) | AI chat, secrets, LLM config, MCP |
+| [services/IntegrationsService/README.md](services/IntegrationsService/README.md) | Google/Bing OAuth, GSC/GA4, keywords |
## Star History
diff --git a/SECURITY.md b/SECURITY.md
index fb009999..1f4ef05c 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -26,4 +26,4 @@ If you find a vulnerability **in Site Audit itself** (e.g. remote code execution
- Run production deployments with strong `POSTGRES_PASSWORD` and `AUTH_SECRET` (see `docker-compose.prod.yml`).
- For client-facing dashboards, set `AUTH_DEFAULT_ROLE=client-readonly` (view reports and use chat) or `viewer` (view reports only). API enforces 403 on mutations; UI hides **Run audit** for read-only roles.
-- Do not commit `.env`, `.secrets/`, or OAuth client secrets. Google credentials are stored in PostgreSQL (`google_app_settings` and per-property columns on `properties`).
+- Do not commit `.env`, `.secrets/`, or OAuth client secrets. Google credentials and API keys are stored in PostgreSQL (`google_app_settings`, `llm_config`, and per-property columns on `properties`). Browser writes to these stores go through the BFF → **AiService** (`PUT /api/secrets`, `PUT /api/llm-config`) — not directly to FastAPI.
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index ee323c2b..a8050b18 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -32,6 +32,7 @@ services:
CHROME_PATH: /usr/bin/chromium
LIGHTHOUSE_PATH: /usr/local/bin/lighthouse
LIGHTHOUSE_CHROME_FLAGS: --headless --no-sandbox --disable-dev-shm-usage --disable-gpu
+ DEPRECATE_PYTHON_INTEGRATIONS: "1"
volumes:
- profiling-data:/data
healthcheck:
@@ -60,12 +61,15 @@ services:
LIGHTHOUSE_PATH: /usr/local/bin/lighthouse
LIGHTHOUSE_CHROME_FLAGS: --headless --no-sandbox --disable-dev-shm-usage --disable-gpu
FILE_SERVICE_URL: http://files:8080
+ AI_SERVICE_URL: http://ai:8092
+ INTEGRATIONS_SERVICE_URL: http://integrations:8093
volumes:
- profiling-data:/data
data:
build:
- context: ./services/Data
+ context: ./services
+ dockerfile: Data/Dockerfile
environment:
DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-website_profiling}
ASPNETCORE_URLS: http://+:8091
@@ -79,9 +83,56 @@ services:
retries: 3
start_period: 15s
+ ai:
+ build:
+ context: ./services
+ dockerfile: AiService/Dockerfile
+ environment:
+ DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-website_profiling}
+ FASTAPI_URL: http://fastapi:8001
+ ASPNETCORE_URLS: http://+:8092
+ WP_MCP_HTTP: "1"
+ depends_on:
+ postgres:
+ condition: service_healthy
+ fastapi:
+ condition: service_started
+ healthcheck:
+ test: ['CMD', 'curl', '-fsS', 'http://127.0.0.1:8092/health']
+ interval: 30s
+ timeout: 5s
+ retries: 3
+ start_period: 20s
+
+ integrations:
+ build:
+ context: ./services
+ dockerfile: IntegrationsService/Dockerfile
+ environment:
+ DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-website_profiling}
+ ASPNETCORE_URLS: http://+:8093
+ FASTAPI_URL: http://fastapi:8001
+ USE_FASTAPI_PYTHON_BRIDGE: "1"
+ ASPNETCORE_ENVIRONMENT: Production
+ AUTH_SECRET: ${AUTH_SECRET:?set AUTH_SECRET}
+ APP_PUBLIC_URL: ${APP_PUBLIC_URL:-}
+ GOOGLE_REDIRECT_URI: ${GOOGLE_REDIRECT_URI:-}
+ depends_on:
+ postgres:
+ condition: service_healthy
+ fastapi:
+ condition: service_started
+ healthcheck:
+ test: ['CMD', 'curl', '-fsS', 'http://127.0.0.1:8093/health']
+ interval: 30s
+ timeout: 5s
+ retries: 3
+ start_period: 20s
+
files:
build:
- context: ./services/FileService
+ context: ./services
+ dockerfile: FileService/Dockerfile
environment:
DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-website_profiling}
REPORT_API_URL: http://fastapi:8001
@@ -101,13 +152,21 @@ services:
condition: service_started
data:
condition: service_healthy
+ ai:
+ condition: service_healthy
+ integrations:
+ condition: service_healthy
ports:
- '${BFF_PORT:-8090}:8090'
environment:
FASTAPI_URL: http://fastapi:8001
FILE_SERVICE_URL: http://files:8080
DATA_SERVICE_URL: http://data:8091
+ AI_SERVICE_URL: http://ai:8092
+ INTEGRATIONS_SERVICE_URL: http://integrations:8093
DATA_ROUTES: "/api/report/meta,/api/report/payload,/api/report/history,/api/report/crawl-payload,/api/report/mobile-delta,/api/report/portfolio,/api/portfolio,/api/issues/status,/api/filters"
+ AI_ROUTES: "/api/chat,/api/links/page-coach,/api/issues/fix-suggestion,/api/issues/action-plan,/api/ai/fix-suggestion,/api/dashboards/ai-generate,/api/content/analyze,/api/content/wizard,/api/llm-config,/api/secrets,/api/ollama/status,/api/report/audit-tool,/api/mcp-tools"
+ INTEGRATIONS_ROUTES: "/api/integrations/google,/api/integrations/bing"
AUTH_SECRET: ${AUTH_SECRET:?set AUTH_SECRET}
AUTH_PASSWORD: ${AUTH_PASSWORD:-}
BFF_ALLOWED_ORIGINS: ${BFF_ALLOWED_ORIGINS:?set BFF_ALLOWED_ORIGINS (the public UI origin)}
@@ -134,28 +193,23 @@ services:
start_period: 30s
mcp:
- image: website-profiling:latest
+ build:
+ context: ./services
+ dockerfile: AiService/Dockerfile
depends_on:
- postgres:
+ ai:
condition: service_healthy
- files:
- condition: service_started
- command: ['/opt/venv/bin/python', '-m', 'website_profiling.mcp.http']
environment:
- WEBSITE_PROFILING_ROOT: /app
DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-website_profiling}
- FILE_SERVICE_URL: http://files:8080
- WP_MCP_HTTP_HOST: 0.0.0.0
- WP_MCP_HTTP_PORT: 8000
- WP_MCP_TOKEN: ${WP_MCP_TOKEN:?set WP_MCP_TOKEN}
- WP_MCP_ALLOWED_HOSTS: ${WP_MCP_ALLOWED_HOSTS:-}
- WP_MCP_ALLOWED_ORIGINS: ${WP_MCP_ALLOWED_ORIGINS:-}
+ FASTAPI_URL: http://fastapi:8001
+ ASPNETCORE_URLS: http://+:8092
+ WP_MCP_HTTP: "1"
WP_MCP_DOMAIN: ${WP_MCP_DOMAIN:-core}
WP_PROPERTY_ID: ${WP_PROPERTY_ID:-}
profiles:
- mcp
ports:
- - '${MCP_PORT:-8000}:8000'
+ - '${MCP_PORT:-8000}:8092'
volumes:
pg-data:
diff --git a/docker-compose.pull.yml b/docker-compose.pull.yml
index 21976f64..246eb6df 100644
--- a/docker-compose.pull.yml
+++ b/docker-compose.pull.yml
@@ -34,6 +34,7 @@ services:
LIGHTHOUSE_PATH: /usr/local/bin/lighthouse
LIGHTHOUSE_CHROME_FLAGS: --headless --no-sandbox --disable-dev-shm-usage --disable-gpu
FASTAPI_ALLOWED_ORIGINS: "http://localhost:8090"
+ DEPRECATE_PYTHON_INTEGRATIONS: "1"
volumes:
- profiling-data:/data
healthcheck:
@@ -62,12 +63,15 @@ services:
LIGHTHOUSE_PATH: /usr/local/bin/lighthouse
LIGHTHOUSE_CHROME_FLAGS: --headless --no-sandbox --disable-dev-shm-usage --disable-gpu
FILE_SERVICE_URL: http://files:8080
+ AI_SERVICE_URL: http://ai:8092
+ INTEGRATIONS_SERVICE_URL: http://integrations:8093
volumes:
- profiling-data:/data
data:
build:
- context: ./services/Data
+ context: ./services
+ dockerfile: Data/Dockerfile
environment:
DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling
ASPNETCORE_URLS: http://+:8091
@@ -81,9 +85,54 @@ services:
retries: 3
start_period: 15s
+ ai:
+ build:
+ context: ./services
+ dockerfile: AiService/Dockerfile
+ environment:
+ DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling
+ FASTAPI_URL: http://fastapi:8001
+ ASPNETCORE_URLS: http://+:8092
+ depends_on:
+ postgres:
+ condition: service_healthy
+ fastapi:
+ condition: service_started
+ healthcheck:
+ test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8092/health"]
+ interval: 30s
+ timeout: 5s
+ retries: 3
+ start_period: 20s
+
+ integrations:
+ build:
+ context: ./services
+ dockerfile: IntegrationsService/Dockerfile
+ environment:
+ DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling
+ ASPNETCORE_URLS: http://+:8093
+ FASTAPI_URL: http://fastapi:8001
+ USE_FASTAPI_PYTHON_BRIDGE: "1"
+ AUTH_SECRET: ${AUTH_SECRET:-}
+ GOOGLE_REDIRECT_URI: ${GOOGLE_REDIRECT_URI:-http://localhost:8090/api/integrations/google/callback}
+ APP_PUBLIC_URL: ${APP_PUBLIC_URL:-http://localhost:3000}
+ depends_on:
+ postgres:
+ condition: service_healthy
+ fastapi:
+ condition: service_started
+ healthcheck:
+ test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8093/health"]
+ interval: 30s
+ timeout: 5s
+ retries: 3
+ start_period: 20s
+
files:
build:
- context: ./services/FileService
+ context: ./services
+ dockerfile: FileService/Dockerfile
environment:
DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling
REPORT_API_URL: http://fastapi:8001
@@ -102,8 +151,12 @@ services:
FASTAPI_URL: http://fastapi:8001
FILE_SERVICE_URL: http://files:8080
DATA_SERVICE_URL: http://data:8091
+ AI_SERVICE_URL: http://ai:8092
+ INTEGRATIONS_SERVICE_URL: http://integrations:8093
BFF_ALLOWED_ORIGINS: "http://localhost:3000"
DATA_ROUTES: "/api/report/meta,/api/report/payload,/api/report/history,/api/report/crawl-payload,/api/report/mobile-delta,/api/report/portfolio,/api/portfolio,/api/issues/status,/api/filters"
+ AI_ROUTES: "/api/chat,/api/links/page-coach,/api/issues/fix-suggestion,/api/issues/action-plan,/api/ai/fix-suggestion,/api/dashboards/ai-generate,/api/content/analyze,/api/content/wizard,/api/llm-config,/api/secrets,/api/ollama/status,/api/report/audit-tool,/api/mcp-tools"
+ INTEGRATIONS_ROUTES: "/api/integrations/google,/api/integrations/bing"
depends_on:
fastapi:
condition: service_started
@@ -111,6 +164,10 @@ services:
condition: service_started
data:
condition: service_healthy
+ ai:
+ condition: service_healthy
+ integrations:
+ condition: service_healthy
web:
image: ${WEB_IMAGE:-website-profiling-web:latest}
diff --git a/docker-compose.yml b/docker-compose.yml
index 21adbe90..329a8a20 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -33,6 +33,7 @@ services:
LIGHTHOUSE_PATH: /usr/local/bin/lighthouse
LIGHTHOUSE_CHROME_FLAGS: --headless --no-sandbox --disable-dev-shm-usage --disable-gpu
FASTAPI_ALLOWED_ORIGINS: "http://localhost:8090"
+ DEPRECATE_PYTHON_INTEGRATIONS: "1"
volumes:
- profiling-data:/data
healthcheck:
@@ -61,6 +62,8 @@ services:
LIGHTHOUSE_PATH: /usr/local/bin/lighthouse
LIGHTHOUSE_CHROME_FLAGS: --headless --no-sandbox --disable-dev-shm-usage --disable-gpu
FILE_SERVICE_URL: http://files:8080
+ AI_SERVICE_URL: http://ai:8092
+ INTEGRATIONS_SERVICE_URL: http://integrations:8093
volumes:
- profiling-data:/data
@@ -68,7 +71,8 @@ services:
# Internal: only the BFF reaches it (no published host port).
data:
build:
- context: ./services/Data
+ context: ./services
+ dockerfile: Data/Dockerfile
environment:
DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling
ASPNETCORE_URLS: http://+:8091
@@ -82,6 +86,51 @@ services:
retries: 3
start_period: 15s
+ ai:
+ build:
+ context: ./services
+ dockerfile: AiService/Dockerfile
+ environment:
+ DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling
+ FASTAPI_URL: http://fastapi:8001
+ ASPNETCORE_URLS: http://+:8092
+ WP_MCP_HTTP: "1"
+ depends_on:
+ postgres:
+ condition: service_healthy
+ fastapi:
+ condition: service_started
+ healthcheck:
+ test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8092/health"]
+ interval: 30s
+ timeout: 5s
+ retries: 3
+ start_period: 20s
+
+ integrations:
+ build:
+ context: ./services
+ dockerfile: IntegrationsService/Dockerfile
+ environment:
+ DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling
+ ASPNETCORE_URLS: http://+:8093
+ FASTAPI_URL: http://fastapi:8001
+ USE_FASTAPI_PYTHON_BRIDGE: "1"
+ AUTH_SECRET: ${AUTH_SECRET:-}
+ GOOGLE_REDIRECT_URI: ${GOOGLE_REDIRECT_URI:-http://localhost:8090/api/integrations/google/callback}
+ APP_PUBLIC_URL: ${APP_PUBLIC_URL:-http://localhost:3000}
+ depends_on:
+ postgres:
+ condition: service_healthy
+ fastapi:
+ condition: service_started
+ healthcheck:
+ test: ["CMD", "curl", "-fsS", "http://127.0.0.1:8093/health"]
+ interval: 30s
+ timeout: 5s
+ retries: 3
+ start_period: 20s
+
# .NET BFF — the single browser-facing API surface (owns auth + CORS).
bff:
build:
@@ -93,10 +142,12 @@ services:
FILE_SERVICE_URL: http://files:8080
BFF_ALLOWED_ORIGINS: "http://localhost:3000"
DATA_SERVICE_URL: http://data:8091
- # Comma-separated /api path prefixes served by the Data service (reads + issue/portfolio mutations).
- # Rollback: remove a prefix here and restart bff.
+ AI_SERVICE_URL: http://ai:8092
+ INTEGRATIONS_SERVICE_URL: http://integrations:8093
+ AUTH_SECRET: ${AUTH_SECRET:-}
DATA_ROUTES: "/api/report/meta,/api/report/payload,/api/report/history,/api/report/crawl-payload,/api/report/mobile-delta,/api/report/portfolio,/api/portfolio,/api/issues/status,/api/filters"
- # AUTH_SECRET: set to enable wp_session auth (must match the web service)
+ AI_ROUTES: "/api/chat,/api/links/page-coach,/api/issues/fix-suggestion,/api/issues/action-plan,/api/ai/fix-suggestion,/api/dashboards/ai-generate,/api/content/analyze,/api/content/wizard,/api/llm-config,/api/secrets,/api/ollama/status,/api/report/audit-tool,/api/mcp-tools"
+ INTEGRATIONS_ROUTES: "/api/integrations/google,/api/integrations/bing"
depends_on:
fastapi:
condition: service_started
@@ -104,6 +155,10 @@ services:
condition: service_started
data:
condition: service_healthy
+ ai:
+ condition: service_healthy
+ integrations:
+ condition: service_healthy
# Vite SPA (nginx). The browser calls the BFF (:8090) for all /api/*.
web:
@@ -126,7 +181,8 @@ services:
# File export service (PDF/Excel). Internal: only the BFF reaches it.
files:
build:
- context: ./services/FileService
+ context: ./services
+ dockerfile: FileService/Dockerfile
environment:
DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling
REPORT_API_URL: http://fastapi:8001
@@ -135,6 +191,12 @@ services:
condition: service_healthy
fastapi:
condition: service_started
+ healthcheck:
+ test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:8080/health || exit 1"]
+ interval: 30s
+ timeout: 5s
+ retries: 3
+ start_period: 15s
volumes:
pg-data:
diff --git a/docs/GLOSSARY.md b/docs/GLOSSARY.md
index f6d11f2d..865ca21e 100644
--- a/docs/GLOSSARY.md
+++ b/docs/GLOSSARY.md
@@ -50,7 +50,7 @@ This glossary maps agency-facing UI terms to internal keys, database tables, and
| Content brief | Keywords Brief button, `/api/keywords/content-brief` | LLM or deterministic | Content planning |
| AI fix suggestions | `llm_recommendation`, `/api/ai/fix-suggestion` | LLM on demand + report build | Actionable remediation |
| AI Chat | `/chat`, `/api/chat`, `chat_sessions` | LLM + read-only audit tools | Conversational audit queries |
-| MCP tools | `python -m website_profiling.mcp` | Same `audit_tools` as chat | IDE integration — see [MCP.md](MCP.md) |
+| MCP tools | AiService MCP (stdio or HTTP) | Same audit tool catalog as chat | IDE integration — see [MCP.md](MCP.md) |
| Read-only session | `AUTH_DEFAULT_ROLE=client-readonly` or `viewer`; `/api/auth/session` returns role and mutation flags | Session cookie | `client-readonly`: view + chat; `viewer`: view only (no chat) |
| Export executive summary | Export view; MCP `export_audit_report` (pdf/csv/json); workbook via Export view or FileService | Report payload + optional AI | Client deliverable |
| ads.txt / security.txt | `site_level`, `get_ads_txt_status`, `get_security_txt_status` | Root file fetch at report build | Publisher / contact file hygiene |
diff --git a/docs/MCP.md b/docs/MCP.md
index 7a41f19d..ea22fcee 100644
--- a/docs/MCP.md
+++ b/docs/MCP.md
@@ -1,8 +1,11 @@
# MCP Server Reference
-Site Audit exposes **340 read-only tools** via the [Model Context Protocol (MCP)](https://modelcontextprotocol.io). Connect from Cursor, Claude Desktop, or any MCP-compatible client to query audit data programmatically.
+Site Audit exposes **369 read-only tools** via the [Model Context Protocol (MCP)](https://modelcontextprotocol.io). The MCP server runs in **AiService** (.NET) — see [services/AiService/README.md](../services/AiService/README.md).
-The same tool catalog powers in-app **AI Chat** at `/chat`.
+- **stdio:** `dotnet run --project services/AiService/src/AiService.Api` with stdio MCP host (see AiService.Mcp)
+- **HTTP:** set `WP_MCP_HTTP=1` on AiService → endpoint `/mcp` (port 8092 by default)
+
+The same tool catalog powers in-app **AI Chat** at `/chat` (also served by AiService via the BFF).
**Related documentation:** [GLOSSARY.md](GLOSSARY.md) · [Documentation index](README.md)
@@ -25,26 +28,36 @@ The same tool catalog powers in-app **AI Chat** at `/chat`.
## Prerequisites
+[.NET SDK 10+](https://dotnet.microsoft.com/download), Postgres, and `./local-run` (or AiService + FastAPI manually). AiService needs FastAPI on `:8001` for the audit-tool bridge until all tools are native C#.
+
```bash
-pip install -r requirements.txt
export DATABASE_URL=postgres://profiling:profiling@localhost:5432/website_profiling # Docker default
# ./local-run default: postgres://postgres:dev@127.0.0.1:5432/website_profiling
-export PYTHONPATH=src
+export FASTAPI_URL=http://127.0.0.1:8001
```
-Start the local stdio server:
+### Local stdio (IDE subprocess)
+
+From the repo root, with Postgres running:
```bash
-python -m website_profiling.mcp
+cd services/AiService
+export DATABASE_URL=postgres://postgres:dev@127.0.0.1:5432/website_profiling
+export FASTAPI_URL=http://127.0.0.1:8001
+dotnet run --project src/AiService.Api
```
+For a dedicated stdio MCP host, use `AddAiServiceMcpStdioHost()` (see `AiService.Mcp/McpServerExtensions.cs`). `./local-run` starts AiService with **`WP_MCP_HTTP=1`** so HTTP MCP is available at **http://localhost:8092/mcp**.
+
For remote access over HTTP, see [Remote Streamable HTTP](#remote-streamable-http).
+> **Legacy:** `python -m website_profiling.mcp` still exists under `src/website_profiling/mcp/` but is no longer the recommended path — use AiService.
+
---
## Domain-scoped servers
-Rather than loading all 340 tools in a single server, Site Audit supports **domain-scoped bundles**. Connect only the domains relevant to your workflow.
+Rather than loading all 369 tools in a single server, Site Audit supports **domain-scoped bundles**. Connect only the domains relevant to your workflow.
| `WP_MCP_DOMAIN` | Tool count | Scope | Recommended use |
|-----------------|------------|-------|-----------------|
@@ -52,9 +65,9 @@ Rather than loading all 340 tools in a single server, Site Audit supports **doma
| `crawl` | Domain subset | Crawl, on-page, schema, accessibility | Technical crawl audits |
| `google` | Domain subset | Google, insight, CTR, keywords | GSC/GA4 analysis |
| `links` | Domain subset | Links, backlinks, indexation | Link architecture |
-| `full` | 340 | All tools | Debugging, legacy single-server setup |
+| `full` | 369 | All tools | Debugging, legacy single-server setup |
-Tier 0 alone includes 16 router/insight tools (`TIER_0_TOOLS` in `tool_domains.py`). Use the `audit://tools` resource or `WP_MCP_DOMAIN=full` for the complete catalog.
+Tier 0 alone includes 16 router/insight tools. Use the `audit://tools` resource or `WP_MCP_DOMAIN=full` for the complete catalog.
Set `WP_PROPERTY_ID` to the default property when tools omit an explicit `property_id` argument.
@@ -64,27 +77,27 @@ Set `WP_PROPERTY_ID` to the default property when tools omit an explicit `proper
### Multi-domain setup (recommended)
-Add to `.cursor/mcp.json` or your MCP client settings:
+Add to `.cursor/mcp.json` or your MCP client settings (stdio — spawn AiService; ensure Postgres and FastAPI are up):
```json
{
"mcpServers": {
"site-audit-core": {
- "command": "python",
- "args": ["-m", "website_profiling.mcp"],
+ "command": "dotnet",
+ "args": ["run", "--project", "services/AiService/src/AiService.Api", "--no-launch-profile"],
"env": {
- "DATABASE_URL": "postgres://profiling:profiling@localhost:5432/website_profiling",
- "PYTHONPATH": "src",
+ "DATABASE_URL": "postgres://postgres:dev@127.0.0.1:5432/website_profiling",
+ "FASTAPI_URL": "http://127.0.0.1:8001",
"WP_MCP_DOMAIN": "core",
"WP_PROPERTY_ID": "1"
}
},
"site-audit-google": {
- "command": "python",
- "args": ["-m", "website_profiling.mcp"],
+ "command": "dotnet",
+ "args": ["run", "--project", "services/AiService/src/AiService.Api", "--no-launch-profile"],
"env": {
- "DATABASE_URL": "postgres://profiling:profiling@localhost:5432/website_profiling",
- "PYTHONPATH": "src",
+ "DATABASE_URL": "postgres://postgres:dev@127.0.0.1:5432/website_profiling",
+ "FASTAPI_URL": "http://127.0.0.1:8001",
"WP_MCP_DOMAIN": "google",
"WP_PROPERTY_ID": "1"
}
@@ -99,11 +112,11 @@ Add to `.cursor/mcp.json` or your MCP client settings:
{
"mcpServers": {
"site-audit": {
- "command": "python",
- "args": ["-m", "website_profiling.mcp"],
+ "command": "dotnet",
+ "args": ["run", "--project", "services/AiService/src/AiService.Api", "--no-launch-profile"],
"env": {
- "DATABASE_URL": "postgres://profiling:profiling@localhost:5432/website_profiling",
- "PYTHONPATH": "src",
+ "DATABASE_URL": "postgres://postgres:dev@127.0.0.1:5432/website_profiling",
+ "FASTAPI_URL": "http://127.0.0.1:8001",
"WP_MCP_DOMAIN": "full",
"WP_PROPERTY_ID": "1"
}
@@ -120,40 +133,38 @@ Use this when Site Audit runs on a hosted server and your MCP client (Cursor, Cl
### Start the HTTP server
-Configure access on **MCP settings** (`/mcp`) in the web UI (recommended), or set environment variables. UI changes apply on the next MCP request without restarting the service.
+Configure access on **MCP settings** (`/mcp`) in the web UI (recommended), or set environment variables on **AiService**. `./local-run` and Docker Compose set `WP_MCP_HTTP=1` on AiService (`:8092`). Production compose can publish host port `8000` → AiService `8092` via the `mcp` profile.
```bash
export DATABASE_URL=postgres://profiling:profiling@localhost:5432/website_profiling
-export PYTHONPATH=src
-export WP_MCP_HTTP_HOST=0.0.0.0
-export WP_MCP_HTTP_PORT=8000
+export FASTAPI_URL=http://127.0.0.1:8001
+export ASPNETCORE_URLS=http://0.0.0.0:8092
+export WP_MCP_HTTP=1
export WP_MCP_DOMAIN=core
export WP_PROPERTY_ID=1
-python -m website_profiling.mcp.http
+cd services/AiService && dotnet run --project src/AiService.Api
```
Set **MCP bearer token** and **Allowed hostnames** on the Secrets page (or via `WP_MCP_TOKEN` / `WP_MCP_ALLOWED_HOSTS`). Environment variables override saved values when set.
-The MCP endpoint is `http://:8000/mcp` by default (`WP_MCP_HTTP_PATH=/mcp`).
+The MCP endpoint is **`http://:8092/mcp`** locally (`8092` is AiService's default port). With `docker-compose.prod.yml --profile mcp`, the host port defaults to **`8000`** mapped to AiService `8092`.
### Environment variables
| Variable | Default | Purpose |
|----------|---------|---------|
-| `WP_MCP_HTTP_HOST` | `127.0.0.1` | Bind address (`0.0.0.0` for Docker) |
-| `WP_MCP_HTTP_PORT` | `8000` | Listen port |
-| `WP_MCP_HTTP_PATH` | `/mcp` | Mount path |
+| `WP_MCP_HTTP` | unset | Set `1` on AiService to expose `/mcp` |
+| `ASPNETCORE_URLS` | `http://+:8092` | AiService bind address |
| `WP_MCP_TOKEN` | unset | Bearer token (**required** when not binding localhost). Save on **Secrets → Remote MCP** or set here (env wins). |
| `WP_MCP_ALLOWED_HOSTS` | unset | Comma-separated `Host` allowlist (**required** for non-localhost bind). Save on **Secrets → Remote MCP** or set here. |
| `WP_MCP_ALLOWED_ORIGINS` | unset | Comma-separated `Origin` allowlist for browser clients |
-| `WP_MCP_JSON_RESPONSE` | `false` | JSON responses instead of SSE streams |
| `WP_MCP_DOMAIN` | `core` | Tool bundle (same as stdio) |
| `WP_PROPERTY_ID` | unset | Default property (same as stdio) |
-**Security:** `WP_MCP_TOKEN` is required when `WP_MCP_HTTP_HOST` is not localhost. Tools are read-only but expose audit, GSC, and GA4 data — treat the token like a database credential.
+**Security:** `WP_MCP_TOKEN` is required when AiService is reachable outside localhost. Tools are read-only but expose audit, GSC, and GA4 data — treat the token like a database credential.
-**DNS rebinding protection:** Whenever a token **and** allowed hosts are configured — via the UI **or** environment variables — the HTTP service enforces the bearer token plus the Host/Origin allowlist in its own middleware, and the MCP SDK's built-in DNS-rebinding check is turned off (the middleware supersedes it). The SDK check only applies as a fallback on a non-localhost bind that has no remote access configured (a state the startup validation otherwise refuses to boot in). Either way, set `WP_MCP_ALLOWED_HOSTS` to the public hostname clients use (e.g. `audit.example.com`).
+**DNS rebinding protection:** When a token **and** allowed hosts are configured — via the UI **or** environment variables — AiService enforces the bearer token plus the Host/Origin allowlist. Set `WP_MCP_ALLOWED_HOSTS` to the public hostname clients use (e.g. `audit.example.com`).
### Cursor / Claude Desktop (remote)
diff --git a/docs/README.md b/docs/README.md
index e8503538..8393a7d9 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -15,8 +15,26 @@ This directory contains product, integration, and operations documentation for *
| [COMPANY_STANDARDS.md](COMPANY_STANDARDS.md) | Agencies / operators | Data classification, crawl scope, security policy |
| [MCP.md](MCP.md) | Integrators | Model Context Protocol server configuration and tool reference |
| [OPS.md](OPS.md) | Operators | Scheduled audits, alerts, migrations, production notes |
+| [services/AiService/README.md](../services/AiService/README.md) | Developers | AI chat, secrets, LLM config, MCP, enrichment (port 8092) |
+| [services/IntegrationsService/README.md](../services/IntegrationsService/README.md) | Developers | Google/Bing OAuth, GSC/GA4 fetch, keywords (port 8093) |
| [services/FileService/README.md](../services/FileService/README.md) | Developers / operators | PDF and Excel workbook export service |
-| `services/Data/` | Developers | .NET read service — report payloads, portfolio, issue status, saved filters |
+| `services/Data/` | Developers | .NET read service — report payloads, portfolio, issue status, saved filters (port 8091) |
+
+---
+
+## API routing (browser)
+
+All `/api/*` calls from the SPA go to the **BFF** (`:8090`). The BFF forwards subsets to:
+
+| Upstream | Examples |
+|----------|----------|
+| **FastAPI** (`:8001`) | `/api/run`, `/api/pipeline-config`, crawl, properties |
+| **IntegrationsService** (`:8093`) | `/api/integrations/google/*`, `/api/integrations/bing/*`, property Google config |
+| **AiService** (`:8092`) | `/api/chat`, `/api/secrets`, `/api/llm-config`, MCP-related APIs |
+| **Data** (`:8091`) | Report payload reads, portfolio, issue status, saved filters |
+| **FileService** (`:8080`) | PDF and Excel export |
+
+See [AGENT.md](../AGENT.md) for the full route split and [services/AiService/README.md](../services/AiService/README.md) for `AI_ROUTES`.
---
diff --git a/scripts/cleanup_junk_properties.py b/scripts/cleanup_junk_properties.py
new file mode 100755
index 00000000..50c35482
--- /dev/null
+++ b/scripts/cleanup_junk_properties.py
@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+"""Remove junk property rows created while typing a Site URL (partial domains)."""
+from __future__ import annotations
+
+import os
+import sys
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+sys.path.insert(0, os.path.join(ROOT, "src"))
+
+from website_profiling.db.pool import db_session
+from website_profiling.db.property_store import delete_property, is_valid_canonical_domain, list_properties_public
+
+
+def main() -> int:
+ dry_run = "--dry-run" in sys.argv
+ with db_session() as conn:
+ props = list_properties_public(conn)
+ junk = [p for p in props if not is_valid_canonical_domain(str(p.get("canonical_domain") or ""))]
+ if not junk:
+ print("No junk properties found.")
+ return 0
+ print(f"{'Would delete' if dry_run else 'Deleting'} {len(junk)} junk propert{'y' if len(junk) == 1 else 'ies'}:")
+ for p in junk:
+ print(f" id={p['id']} domain={p.get('canonical_domain')!r}")
+ if dry_run:
+ return 0
+ deleted = 0
+ for p in junk:
+ if delete_property(conn, int(p["id"])):
+ deleted += 1
+ print(f"Deleted {deleted} properties.")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/local-prod.sh b/scripts/local-prod.sh
index eceb7ef3..655f5104 100755
--- a/scripts/local-prod.sh
+++ b/scripts/local-prod.sh
@@ -1,13 +1,15 @@
#!/usr/bin/env bash
-# Local prod: same Postgres as ./local-run, Vite build + preview (NODE_ENV=production).
+# Local prod: same Postgres as ./local-run, full stack + Vite build + preview (NODE_ENV=production).
# Usage: ./local-prod [command]
-# (default) start — DB, migrations, npm run build, npm run preview
+# (default) start — DB, migrations, .NET stack, worker, FastAPI, vite preview
# build — npm run build only
# help — show commands
set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT"
+# shellcheck source=scripts/local-run-common.sh
+source "$ROOT/scripts/local-run-common.sh"
PG_CONTAINER="${WP_PG_CONTAINER:-wp-pg}"
PG_PORT="${WP_PG_PORT:-5432}"
@@ -21,6 +23,9 @@ export PYTHON="${PYTHON:-$ROOT/.venv/bin/python}"
export WEBSITE_PROFILING_ROOT="$ROOT"
export PYTHONPATH="${PYTHONPATH:+$PYTHONPATH:}$ROOT/src"
export NODE_ENV=production
+export VITE_BFF_BASE_URL="${VITE_BFF_BASE_URL:-http://localhost:8090}"
+export DEPRECATE_PYTHON_INTEGRATIONS="${DEPRECATE_PYTHON_INTEGRATIONS:-1}"
+export USE_FASTAPI_PYTHON_BRIDGE="${USE_FASTAPI_PYTHON_BRIDGE:-1}"
WEB="$ROOT/web"
LOCAL_RUN="$ROOT/scripts/local-run.sh"
@@ -55,10 +60,12 @@ wait_for_pid() {
stop_service() {
local name="$1"
local pid="$2"
+ local port="${3:-}"
[[ -z "$pid" ]] && return 0
if ! kill -0 "$pid" 2>/dev/null; then
wait "$pid" 2>/dev/null || true
log "$name already stopped."
+ [[ -n "$port" ]] && free_port "$port"
return 0
fi
log "Stopping $name (PID $pid)..."
@@ -70,6 +77,7 @@ stop_service() {
fi
wait "$pid" 2>/dev/null || true
log "$name stopped."
+ [[ -n "$port" ]] && free_port "$port"
}
disown_bg() {
@@ -107,8 +115,8 @@ cmd_web_deps() {
cmd_build() {
cmd_web_deps
- log "Building Vite SPA (production)"
- (cd "$WEB" && npm run build)
+ log "Building Vite SPA (production, VITE_BFF_BASE_URL=$VITE_BFF_BASE_URL)"
+ (cd "$WEB" && VITE_BFF_BASE_URL="$VITE_BFF_BASE_URL" npm run build)
}
cmd_start() {
@@ -119,6 +127,8 @@ cmd_start() {
esac
done
+ need_cmd dotnet
+
mkdir -p "$DATA_DIR"
log "Ensuring Postgres and migrations (via ./local-run migrate)"
"$LOCAL_RUN" migrate
@@ -128,13 +138,14 @@ cmd_start() {
cmd_web_deps
log "Skipping build (--skip-build)"
fi
- log "Starting Vite preview server (Ctrl+C stops all services including Postgres)"
+ log "Starting local prod stack (Ctrl+C stops all services including Postgres)"
log "DATABASE_URL=$DATABASE_URL"
log "DATA_DIR=$DATA_DIR"
- log "PYTHON=$PYTHON"
- log "NODE_ENV=$NODE_ENV"
+ log "VITE_BFF_BASE_URL=$VITE_BFF_BASE_URL"
+ log "INTEGRATIONS_SERVICE_URL=${INTEGRATIONS_SERVICE_URL:-http://127.0.0.1:8093}"
cd "$ROOT"
export DATABASE_URL DATA_DIR PYTHON WEBSITE_PROFILING_ROOT PYTHONPATH NODE_ENV
+ export VITE_BFF_BASE_URL DEPRECATE_PYTHON_INTEGRATIONS USE_FASTAPI_PYTHON_BRIDGE
WORKER_PID=""
UVICORN_PID=""
@@ -153,7 +164,8 @@ cmd_start() {
log "Shutting down local prod stack..."
stop_service "Vite preview" "$NPM_PID"
NPM_PID=""
- stop_service "FastAPI" "$UVICORN_PID"
+ stop_host_dotnet_stack stop_service
+ stop_service "FastAPI" "$UVICORN_PID" 8001
UVICORN_PID=""
stop_service "pipeline worker" "$WORKER_PID"
WORKER_PID=""
@@ -163,6 +175,15 @@ cmd_start() {
}
trap cleanup_prod EXIT INT TERM
+ start_host_dotnet_base "$ROOT" Production
+ disown_bg "$FILE_SERVICE_PID"
+ disown_bg "$DATA_PID"
+ disown_bg "$AI_PID"
+
+ export AI_SERVICE_URL="${AI_SERVICE_URL:-http://127.0.0.1:8092}"
+ export INTEGRATIONS_SERVICE_URL="${INTEGRATIONS_SERVICE_URL:-http://127.0.0.1:8093}"
+ export FILE_SERVICE_URL="${FILE_SERVICE_URL:-http://127.0.0.1:8080}"
+
log "Starting pipeline worker"
"$ROOT/.venv/bin/python" -m website_profiling.worker &
WORKER_PID=$!
@@ -170,10 +191,16 @@ cmd_start() {
log "Starting FastAPI on port 8001"
export FASTAPI_URL="http://127.0.0.1:8001"
+ export FASTAPI_ALLOWED_ORIGINS="http://localhost:8090"
"$ROOT/.venv/bin/uvicorn" website_profiling.api.main:app \
--host 0.0.0.0 --port 8001 --workers 1 &
UVICORN_PID=$!
disown_bg "$UVICORN_PID"
+ wait_for_http "http://127.0.0.1:8001/api/health" "FastAPI" 90 || die "FastAPI failed to start"
+
+ start_host_integrations_bff "$ROOT" Production
+ disown_bg "$INTEGRATIONS_PID"
+ disown_bg "$BFF_PID"
cd "$WEB"
npm run preview -- --host 0.0.0.0 --port 3000 &
@@ -186,10 +213,10 @@ cmd_start() {
cmd_help() {
cat </data)
- AUTH_SECRET (optional — enables login when set)
+ VITE_BFF_BASE_URL (default: http://localhost:8090 — baked into the SPA at build time)
+ AUTH_SECRET (optional — required for Google OAuth in Production ASP.NET mode)
+ GOOGLE_REDIRECT_URI, APP_PUBLIC_URL (Google OAuth callback + post-login redirect)
WP_PG_CONTAINER, WP_PG_PORT, WP_PG_PASSWORD, WP_PG_DB
After start, open: http://localhost:3000/home
Use localhost (not 127.0.0.1) for pipeline APIs.
Dev mode with hot reload: ./local-run start
+Docker prod layout: docker compose -f docker-compose.prod.yml up
EOF
}
diff --git a/scripts/local-run-common.sh b/scripts/local-run-common.sh
new file mode 100644
index 00000000..855b3d44
--- /dev/null
+++ b/scripts/local-run-common.sh
@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+# Shared helpers for local-run.sh and local-prod.sh (source, do not execute directly).
+
+free_port() {
+ local port="$1"
+ local pids
+ pids="$(lsof -nP -tiTCP:"$port" -sTCP:LISTEN 2>/dev/null || true)"
+ if [[ -n "$pids" ]]; then
+ printf '\033[1;33m!\033[0m Stopping stale listener on port %s (PID(s): %s)\n' "$port" "${pids//$'\n'/ }" >&2
+ # shellcheck disable=SC2086
+ kill $pids 2>/dev/null || true
+ sleep 0.3
+ fi
+}
+
+wait_for_http() {
+ local url="$1"
+ local name="$2"
+ local timeout="${3:-60}"
+ local i
+ for ((i = 0; i < timeout; i++)); do
+ if curl -fsS "$url" >/dev/null 2>&1; then
+ return 0
+ fi
+ sleep 1
+ done
+ printf '\033[1;31m✗\033[0m %s did not become ready at %s\n' "$name" "$url" >&2
+ return 1
+}
+
+start_host_dotnet_base() {
+ local root="$1"
+ local mode="${2:-Development}"
+
+ export REPORT_API_URL="${REPORT_API_URL:-http://127.0.0.1:8001}"
+ export AI_SERVICE_URL="${AI_SERVICE_URL:-http://127.0.0.1:8092}"
+ export INTEGRATIONS_SERVICE_URL="${INTEGRATIONS_SERVICE_URL:-http://127.0.0.1:8093}"
+ export FILE_SERVICE_URL="${FILE_SERVICE_URL:-http://127.0.0.1:8080}"
+
+ free_port 8080
+ printf '\033[1;36m→\033[0m Starting FileService on port 8080\n'
+ (cd "$root/services/FileService" && \
+ ASPNETCORE_URLS="http://127.0.0.1:8080" \
+ ASPNETCORE_ENVIRONMENT="$mode" \
+ dotnet run --project src/FileService.Api --no-launch-profile) &
+ FILE_SERVICE_PID=$!
+
+ free_port 8091
+ printf '\033[1;36m→\033[0m Starting Data service on port 8091\n'
+ (cd "$root/services/Data" && \
+ DATABASE_URL="$DATABASE_URL" \
+ ASPNETCORE_URLS="http://127.0.0.1:8091" \
+ ASPNETCORE_ENVIRONMENT="$mode" \
+ dotnet run --project src/Data.Api --no-launch-profile) &
+ DATA_PID=$!
+
+ free_port 8092
+ printf '\033[1;36m→\033[0m Starting AiService on port 8092\n'
+ (cd "$root/services/AiService" && \
+ DATABASE_URL="$DATABASE_URL" \
+ FASTAPI_URL="http://127.0.0.1:8001" \
+ ASPNETCORE_URLS="http://127.0.0.1:8092" \
+ ASPNETCORE_ENVIRONMENT="$mode" \
+ WP_MCP_HTTP=1 \
+ dotnet run --project src/AiService.Api --no-launch-profile) &
+ AI_PID=$!
+
+ wait_for_http "http://127.0.0.1:8080/health" "FileService"
+ wait_for_http "http://127.0.0.1:8091/health" "Data service"
+ wait_for_http "http://127.0.0.1:8092/health" "AiService"
+}
+
+start_host_integrations_bff() {
+ local root="$1"
+ local mode="${2:-Development}"
+
+ free_port 8093
+ printf '\033[1;36m→\033[0m Starting IntegrationsService on port 8093\n'
+ (cd "$root/services/IntegrationsService" && \
+ DATABASE_URL="$DATABASE_URL" \
+ FASTAPI_URL="http://127.0.0.1:8001" \
+ USE_FASTAPI_PYTHON_BRIDGE="${USE_FASTAPI_PYTHON_BRIDGE:-1}" \
+ ASPNETCORE_URLS="http://127.0.0.1:8093" \
+ ASPNETCORE_ENVIRONMENT="$mode" \
+ AUTH_SECRET="${AUTH_SECRET:-}" \
+ SESSION_SECRET="${SESSION_SECRET:-}" \
+ GOOGLE_REDIRECT_URI="${GOOGLE_REDIRECT_URI:-http://localhost:8090/api/integrations/google/callback}" \
+ APP_PUBLIC_URL="${APP_PUBLIC_URL:-http://localhost:3000}" \
+ dotnet run --project src/IntegrationsService.Api --no-launch-profile) &
+ INTEGRATIONS_PID=$!
+ wait_for_http "http://127.0.0.1:8093/health" "IntegrationsService"
+
+ free_port 8090
+ printf '\033[1;36m→\033[0m Starting BFF on port 8090\n'
+ (cd "$root/services/Bff" && \
+ FASTAPI_URL="http://127.0.0.1:8001" \
+ FILE_SERVICE_URL="$FILE_SERVICE_URL" \
+ DATA_SERVICE_URL="http://127.0.0.1:8091" \
+ AI_SERVICE_URL="$AI_SERVICE_URL" \
+ INTEGRATIONS_SERVICE_URL="$INTEGRATIONS_SERVICE_URL" \
+ DATA_ROUTES="${DATA_ROUTES:-/api/report/meta,/api/report/payload,/api/report/history,/api/report/crawl-payload,/api/report/mobile-delta,/api/report/portfolio,/api/portfolio,/api/issues/status,/api/filters}" \
+ AI_ROUTES="${AI_ROUTES:-/api/chat,/api/links/page-coach,/api/issues/fix-suggestion,/api/issues/action-plan,/api/ai/fix-suggestion,/api/dashboards/ai-generate,/api/content/analyze,/api/content/wizard,/api/llm-config,/api/secrets,/api/ollama/status,/api/report/audit-tool,/api/mcp-tools}" \
+ INTEGRATIONS_ROUTES="${INTEGRATIONS_ROUTES:-/api/integrations/google,/api/integrations/bing}" \
+ BFF_ALLOWED_ORIGINS="${BFF_ALLOWED_ORIGINS:-http://localhost:3000}" \
+ AUTH_SECRET="${AUTH_SECRET:-}" \
+ SESSION_SECRET="${SESSION_SECRET:-}" \
+ AUTH_PASSWORD="${AUTH_PASSWORD:-}" \
+ AUTH_USER="${AUTH_USER:-}" \
+ ASPNETCORE_URLS="http://127.0.0.1:8090" \
+ ASPNETCORE_ENVIRONMENT="$mode" \
+ dotnet run --project src/Bff.Api --no-launch-profile) &
+ BFF_PID=$!
+ wait_for_http "http://127.0.0.1:8090/health" "BFF"
+}
+
+start_host_dotnet_stack() {
+ local root="$1"
+ local mode="${2:-Development}"
+ start_host_dotnet_base "$root" "$mode"
+ start_host_integrations_bff "$root" "$mode"
+}
+
+stop_host_dotnet_stack() {
+ local stop_service_fn="$1"
+ "$stop_service_fn" "BFF" "${BFF_PID:-}" 8090
+ BFF_PID=""
+ "$stop_service_fn" "IntegrationsService" "${INTEGRATIONS_PID:-}" 8093
+ INTEGRATIONS_PID=""
+ "$stop_service_fn" "AiService" "${AI_PID:-}" 8092
+ AI_PID=""
+ "$stop_service_fn" "Data" "${DATA_PID:-}" 8091
+ DATA_PID=""
+ "$stop_service_fn" "FileService" "${FILE_SERVICE_PID:-}" 8080
+ FILE_SERVICE_PID=""
+}
diff --git a/scripts/local-run.sh b/scripts/local-run.sh
index b911562b..81580ba3 100755
--- a/scripts/local-run.sh
+++ b/scripts/local-run.sh
@@ -12,6 +12,8 @@ set -euo pipefail
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT"
+# shellcheck source=scripts/local-run-common.sh
+source "$ROOT/scripts/local-run-common.sh"
PG_CONTAINER="${WP_PG_CONTAINER:-wp-pg}"
PG_IMAGE="${WP_PG_IMAGE:-postgres:16-alpine}"
@@ -33,19 +35,6 @@ log() { printf '\033[1;36m→\033[0m %s\n' "$*"; }
warn() { printf '\033[1;33m!\033[0m %s\n' "$*" >&2; }
die() { printf '\033[1;31m✗\033[0m %s\n' "$*" >&2; exit 1; }
-# Kill any process still listening on a TCP port (stale dev servers after Ctrl+C).
-free_port() {
- local port="$1"
- local pids
- pids="$(lsof -nP -tiTCP:"$port" -sTCP:LISTEN 2>/dev/null || true)"
- if [[ -n "$pids" ]]; then
- warn "Stopping stale listener on port $port (PID(s): ${pids//$'\n'/ })"
- # shellcheck disable=SC2086
- kill $pids 2>/dev/null || true
- sleep 0.3
- fi
-}
-
# Send signal to a process and its descendants (dotnet/npm subshell trees).
kill_process_tree() {
local pid="$1"
@@ -122,6 +111,23 @@ wait_for_postgres() {
die "Postgres did not become ready in time (container: $PG_CONTAINER)"
}
+wait_for_http_logged() {
+ local url="$1"
+ local label="$2"
+ local timeout="${3:-90}"
+ local i
+ need_cmd curl
+ log "Waiting for $label ($url)"
+ for ((i = 0; i < timeout; i++)); do
+ if curl -fsS "$url" >/dev/null 2>&1; then
+ log "$label ready"
+ return 0
+ fi
+ sleep 1
+ done
+ die "$label did not become ready in ${timeout}s ($url)"
+}
+
cmd_db() {
ensure_docker
if docker ps -a --format '{{.Names}}' | grep -qx "$PG_CONTAINER"; then
@@ -209,7 +215,9 @@ cmd_start() {
UVICORN_PID=""
FILE_SERVICE_PID=""
DATA_PID=""
+ AI_PID=""
BFF_PID=""
+ INTEGRATIONS_PID=""
_CLEANUP_DONE=0
set +m
@@ -223,16 +231,13 @@ cmd_start() {
log "Shutting down local dev stack..."
# Reverse startup order; Vite (foreground) is already exiting from Ctrl+C.
- stop_service "BFF" "$BFF_PID" 8090
- BFF_PID=""
- stop_service "Data" "$DATA_PID" 8091
- DATA_PID=""
+ if command -v dotnet >/dev/null 2>&1; then
+ stop_host_dotnet_stack stop_service
+ fi
stop_service "FastAPI" "$UVICORN_PID" 8001
UVICORN_PID=""
stop_service "pipeline worker" "$WORKER_PID"
WORKER_PID=""
- stop_service "FileService" "$FILE_SERVICE_PID" 8080
- FILE_SERVICE_PID=""
stop_postgres
log "All services stopped."
exit 0
@@ -240,30 +245,21 @@ cmd_start() {
trap cleanup_local EXIT INT TERM
if command -v dotnet >/dev/null 2>&1; then
- free_port 8080
- log "Starting FileService on port 8080"
- export REPORT_API_URL="http://127.0.0.1:8001"
- (cd "$ROOT/services/FileService" && \
- ASPNETCORE_URLS="http://127.0.0.1:8080" \
- ASPNETCORE_ENVIRONMENT=Development \
- dotnet run --project src/FileService.Api --no-launch-profile) &
- FILE_SERVICE_PID=$!
+ start_host_dotnet_base "$ROOT" Development
disown_bg "$FILE_SERVICE_PID"
-
- free_port 8091
- log "Starting Data service on port 8091"
- (cd "$ROOT/services/Data" && \
- DATABASE_URL="$DATABASE_URL" \
- ASPNETCORE_URLS="http://127.0.0.1:8091" \
- ASPNETCORE_ENVIRONMENT=Development \
- dotnet run --project src/Data.Api --no-launch-profile) &
- DATA_PID=$!
disown_bg "$DATA_PID"
+ disown_bg "$AI_PID"
else
warn "dotnet not found — PDF export requires FileService (see services/FileService/README.md)"
warn "dotnet not found — Data service unavailable on port 8091"
+ warn "dotnet not found — AiService unavailable on port 8092"
+ warn "dotnet not found — IntegrationsService unavailable on port 8093"
fi
+ export AI_SERVICE_URL="${AI_SERVICE_URL:-http://127.0.0.1:8092}"
+ export INTEGRATIONS_SERVICE_URL="${INTEGRATIONS_SERVICE_URL:-http://127.0.0.1:8093}"
+ export FILE_SERVICE_URL="${FILE_SERVICE_URL:-http://127.0.0.1:8080}"
+
log "Starting pipeline worker"
"$VENV/bin/python" -m website_profiling.worker &
WORKER_PID=$!
@@ -273,24 +269,16 @@ cmd_start() {
log "Starting FastAPI on port 8001"
export FASTAPI_URL="http://127.0.0.1:8001"
export FASTAPI_ALLOWED_ORIGINS="http://localhost:8090"
+ export DEPRECATE_PYTHON_INTEGRATIONS="${DEPRECATE_PYTHON_INTEGRATIONS:-1}"
"$VENV/bin/uvicorn" website_profiling.api.main:app \
--host 0.0.0.0 --port 8001 --workers 1 &
UVICORN_PID=$!
disown_bg "$UVICORN_PID"
+ wait_for_http_logged "http://127.0.0.1:8001/api/health" "FastAPI"
if command -v dotnet >/dev/null 2>&1; then
- free_port 8090
- log "Starting BFF on port 8090"
- (cd "$ROOT/services/Bff" && \
- FASTAPI_URL="http://127.0.0.1:8001" \
- FILE_SERVICE_URL="${FILE_SERVICE_URL:-http://127.0.0.1:8080}" \
- DATA_SERVICE_URL="http://127.0.0.1:8091" \
- DATA_ROUTES="${DATA_ROUTES:-/api/report/meta,/api/report/payload,/api/report/history,/api/report/crawl-payload,/api/report/mobile-delta,/api/report/portfolio,/api/portfolio,/api/issues/status,/api/filters}" \
- BFF_ALLOWED_ORIGINS="http://localhost:3000" \
- ASPNETCORE_URLS="http://127.0.0.1:8090" \
- ASPNETCORE_ENVIRONMENT=Development \
- dotnet run --project src/Bff.Api --no-launch-profile) &
- BFF_PID=$!
+ start_host_integrations_bff "$ROOT" Development
+ disown_bg "$INTEGRATIONS_PID"
disown_bg "$BFF_PID"
else
warn "dotnet not found — browser API calls need the BFF (see services/Bff/)"
@@ -359,6 +347,9 @@ Environment overrides (optional):
DATABASE_URL (default: postgres://postgres:dev@127.0.0.1:5432/website_profiling)
DATA_DIR (default: /data)
DATA_ROUTES (default: report reads, portfolio, issues status, saved filters)
+ INTEGRATIONS_ROUTES (default: /api/integrations/google,/api/integrations/bing)
+ AUTH_SECRET, GOOGLE_REDIRECT_URI, APP_PUBLIC_URL (Google OAuth)
+ DEPRECATE_PYTHON_INTEGRATIONS (default: 1 — Python integration routes return 410)
WP_PG_CONTAINER, WP_PG_PORT, WP_PG_PASSWORD, WP_PG_DB
After start, open: http://localhost:3000/home
diff --git a/scripts/local-test.sh b/scripts/local-test.sh
index dd2c1d7c..240d7c7d 100755
--- a/scripts/local-test.sh
+++ b/scripts/local-test.sh
@@ -332,6 +332,7 @@ steps_dotnet() {
return 0
fi
run_step "dotnet test Data (services/Data/Data.slnx)" dotnet_test_sln "Data" "Data.slnx"
+ run_step "dotnet test AiService (services/AiService/AiService.slnx)" dotnet_test_sln "AiService" "AiService.slnx"
run_step "dotnet test Bff (services/Bff/Bff.slnx)" dotnet_test_sln "Bff" "Bff.slnx"
run_step_or_skip_openapi
run_step "dotnet test FileService (services/FileService/FileService.slnx)" dotnet_test_sln "FileService" "FileService.slnx"
diff --git a/services/.dockerignore b/services/.dockerignore
new file mode 100644
index 00000000..ab41d99e
--- /dev/null
+++ b/services/.dockerignore
@@ -0,0 +1,4 @@
+**/bin/
+**/obj/
+**/.vs/
+**/TestResults/
diff --git a/services/AiService/.dockerignore b/services/AiService/.dockerignore
new file mode 100644
index 00000000..ab41d99e
--- /dev/null
+++ b/services/AiService/.dockerignore
@@ -0,0 +1,4 @@
+**/bin/
+**/obj/
+**/.vs/
+**/TestResults/
diff --git a/services/AiService/AiService.slnx b/services/AiService/AiService.slnx
new file mode 100644
index 00000000..f4558068
--- /dev/null
+++ b/services/AiService/AiService.slnx
@@ -0,0 +1,13 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/services/AiService/Dockerfile b/services/AiService/Dockerfile
new file mode 100644
index 00000000..54711f16
--- /dev/null
+++ b/services/AiService/Dockerfile
@@ -0,0 +1,24 @@
+FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build
+WORKDIR /src
+COPY Shared/WebsiteProfiling.Contracts/WebsiteProfiling.Contracts.csproj Shared/WebsiteProfiling.Contracts/
+COPY AiService/AiService.slnx AiService/
+COPY AiService/src/AiService.Domain/AiService.Domain.csproj AiService/src/AiService.Domain/
+COPY AiService/src/AiService.Providers/AiService.Providers.csproj AiService/src/AiService.Providers/
+COPY AiService/src/AiService.Tools/AiService.Tools.csproj AiService/src/AiService.Tools/
+COPY AiService/src/AiService.Application/AiService.Application.csproj AiService/src/AiService.Application/
+COPY AiService/src/AiService.Mcp/AiService.Mcp.csproj AiService/src/AiService.Mcp/
+COPY AiService/src/AiService.Api/AiService.Api.csproj AiService/src/AiService.Api/
+RUN dotnet restore AiService/src/AiService.Api/AiService.Api.csproj
+COPY Shared/WebsiteProfiling.Contracts/ Shared/WebsiteProfiling.Contracts/
+COPY AiService/src/ AiService/src/
+RUN dotnet publish AiService/src/AiService.Api/AiService.Api.csproj -c Release -o /app/publish --no-restore
+
+FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS runtime
+WORKDIR /app
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends curl \
+ && rm -rf /var/lib/apt/lists/*
+ENV ASPNETCORE_URLS=http://+:8092
+EXPOSE 8092
+COPY --from=build /app/publish .
+ENTRYPOINT ["dotnet", "AiService.Api.dll"]
diff --git a/services/AiService/README.md b/services/AiService/README.md
new file mode 100644
index 00000000..b79bfc0b
--- /dev/null
+++ b/services/AiService/README.md
@@ -0,0 +1,87 @@
+# AiService
+
+Standalone .NET microservice for **all AI/LLM functionality** in Site Audit, built on [Microsoft.Extensions.AI](https://www.nuget.org/packages/Microsoft.Extensions.AI).
+
+Python FastAPI retains crawl, pipeline, and integrations. AiService owns chat, structured completions, enrichment, **secrets and LLM config writes**, MCP, and audit-tool dispatch for the chat agent.
+
+## Run locally
+
+Prerequisites: [.NET SDK 10+](https://dotnet.microsoft.com/download), Postgres, FastAPI on port 8001 (audit-tool bridge for tools not yet native in C#).
+
+```bash
+cd services/AiService
+export DATABASE_URL=postgres://postgres:dev@127.0.0.1:5432/website_profiling
+export FASTAPI_URL=http://127.0.0.1:8001
+dotnet run --project src/AiService.Api
+```
+
+Service listens on **http://localhost:8092**. Swagger UI in Development: **http://localhost:8092/docs**.
+
+`./local-run` starts AiService automatically (with `WP_MCP_HTTP=1`). The BFF routes browser AI requests here via `AI_SERVICE_URL` and `AI_ROUTES`.
+
+## BFF routing (`AI_ROUTES`)
+
+Browser clients never call AiService directly. The BFF proxies these paths (see `docker-compose.yml` / `services/Bff/src/Bff.Api/appsettings.Development.json`):
+
+| Path | Purpose |
+|------|---------|
+| `/api/chat` | Property-scoped chat (SSE) |
+| `/api/llm-config` | LLM provider/model and risk toggles |
+| `/api/secrets` | API keys, Google OAuth app credentials, MCP token |
+| `/api/ollama/status` | Local Ollama catalog |
+| `/api/issues/fix-suggestion`, `/api/issues/action-plan` | Issue AI helpers |
+| `/api/ai/fix-suggestion` | Legacy fix-suggestion alias |
+| `/api/dashboards/ai-generate` | Dashboard AI |
+| `/api/content/analyze`, `/api/content/wizard` | Content studio AI |
+| `/api/links/page-coach` | Page coach |
+| `/api/report/audit-tool` | Internal audit-tool dispatch (also used by tool bridge) |
+| `/api/mcp-tools` | MCP tool catalog API |
+
+## Secrets and config stores
+
+`SecretsService` unifies **GET/PUT `/api/secrets`** across Postgres tables:
+
+| Store | Examples |
+|-------|----------|
+| `llm_config` | `OPENAI_API_KEY`, `llm_enabled`, provider/model |
+| `pipeline_config` | Integration keys routed from Secrets UI (e.g. `PERPLEXITY_API_KEY`) |
+| `google_app_settings` | Google OAuth client ID/secret (app-wide) |
+
+`LlmConfigRepository` **merges** partial PUT payloads so Risk Settings toggles do not wipe API keys. Per-property Google tokens remain on `properties` (Integrations UI).
+
+## Environment variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `DATABASE_URL` | — | Postgres (`llm_config`, `llm_cache`, `chat_sessions`, `pipeline_config`, `google_app_settings`, `report_payload`) |
+| `FASTAPI_URL` | `http://127.0.0.1:8001` | Python audit-tool bridge for unported tools |
+| `AI_SERVICE_URL` | — | Used by Python worker (`llm_client_http.py`) |
+| `ASPNETCORE_URLS` | `http://+:8092` | Bind address |
+| `WP_MCP_HTTP` | — | Set `1` to expose MCP at `/mcp` |
+| `WP_MCP_DOMAIN` | `core` | MCP tool bundle: core, crawl, google, links, full |
+
+## Architecture
+
+```
+BFF (:8090) → AiService (:8092)
+ ├─ Microsoft.Extensions.AI (OpenAI, Groq, Ollama, Anthropic, Gemini)
+ ├─ SecretsService → llm_config / pipeline_config / google_app_settings
+ ├─ ToolDispatcher → native C# handlers + Python bridge (369 tools)
+ └─ MCP (stdio or HTTP on /mcp)
+Python worker → POST /internal/enrichment/* and llm_client_http → AiService
+```
+
+## Projects
+
+| Project | Role |
+|---------|------|
+| `AiService.Api` | HTTP controllers (paths match former FastAPI routes) |
+| `AiService.Application` | Services, repos, chat agent, enrichment, secrets |
+| `AiService.Providers` | `IChatClientFactory`, structured JSON completions |
+| `AiService.Tools` | Audit tool catalog, dispatch, payload slices |
+| `AiService.Mcp` | Model Context Protocol server |
+| `AiService.Domain` | Entities and repository interfaces |
+
+## OpenAPI
+
+AiService exposes Swagger at `/docs` in Development. The web app's `web/openapi.json` is generated from **FastAPI only** — it does not include AiService routes. Use AiService Swagger or BFF proxy paths when integrating.
diff --git a/services/AiService/src/AiService.Api/AiService.Api.csproj b/services/AiService/src/AiService.Api/AiService.Api.csproj
new file mode 100644
index 00000000..444cc79c
--- /dev/null
+++ b/services/AiService/src/AiService.Api/AiService.Api.csproj
@@ -0,0 +1,14 @@
+
+
+
+ net10.0
+ enable
+ enable
+
+
+
+
+
+
+
+
diff --git a/services/AiService/src/AiService.Api/Controllers/AuditToolController.cs b/services/AiService/src/AiService.Api/Controllers/AuditToolController.cs
new file mode 100644
index 00000000..dab3a721
--- /dev/null
+++ b/services/AiService/src/AiService.Api/Controllers/AuditToolController.cs
@@ -0,0 +1,56 @@
+using System.Text.Json.Nodes;
+using AiService.Tools.Registry;
+using Microsoft.AspNetCore.Mvc;
+
+namespace AiService.Api.Controllers;
+
+/// Audit tool dispatch — POST /api/report/audit-tool.
+[ApiController]
+[Route("api/report")]
+[Tags("Report Audit Tool")]
+public sealed class AuditToolController : ControllerBase
+{
+ private readonly ToolDispatcher _dispatcher;
+
+ public AuditToolController(ToolDispatcher dispatcher) => _dispatcher = dispatcher;
+
+ [HttpPost("audit-tool")]
+ [ProducesResponseType(StatusCodes.Status200OK)]
+ [ProducesResponseType(StatusCodes.Status400BadRequest)]
+ [ProducesResponseType(StatusCodes.Status500InternalServerError)]
+ public async Task Run([FromBody] AuditToolBody body, CancellationToken cancellationToken)
+ {
+ if (string.IsNullOrWhiteSpace(body.ToolName) || body.PropertyId == 0)
+ {
+ return BadRequest(new { detail = "toolName and propertyId required" });
+ }
+
+ try
+ {
+ var args = body.Args ?? [];
+ var result = await _dispatcher.DispatchAsync(
+ body.ToolName,
+ body.PropertyId,
+ body.ReportId,
+ args,
+ cancellationToken);
+
+ return Ok(new { result });
+ }
+ catch (Exception ex)
+ {
+ return StatusCode(StatusCodes.Status500InternalServerError, new { detail = ex.Message });
+ }
+ }
+
+ public sealed class AuditToolBody
+ {
+ public string ToolName { get; set; } = "";
+
+ public int PropertyId { get; set; }
+
+ public int? ReportId { get; set; }
+
+ public JsonObject? Args { get; set; }
+ }
+}
diff --git a/services/AiService/src/AiService.Api/Controllers/ChatController.cs b/services/AiService/src/AiService.Api/Controllers/ChatController.cs
new file mode 100644
index 00000000..0baf2335
--- /dev/null
+++ b/services/AiService/src/AiService.Api/Controllers/ChatController.cs
@@ -0,0 +1,258 @@
+using System.Text.Json.Nodes;
+using System.Threading.Channels;
+using AiService.Application.Chat;
+using AiService.Application.Dto;
+using AiService.Application.Services;
+using AiService.Domain.Repositories;
+using AiService.Tools.Context;
+using AiService.Tools.Options;
+using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Options;
+
+namespace AiService.Api.Controllers;
+
+///
+/// Chat agent and session endpoints ported from FastAPI's /api/chat/*.
+///
+[ApiController]
+[Route("api/chat")]
+[Tags("Chat")]
+public sealed class ChatController : ControllerBase
+{
+ private readonly IChatSessionRepository _sessions;
+ private readonly ChatAgentService _agent;
+ private readonly IHttpClientFactory _httpClientFactory;
+ private readonly IOptions _fastApiOptions;
+ private readonly ILogger _logger;
+
+ public ChatController(
+ IChatSessionRepository sessions,
+ ChatAgentService agent,
+ IHttpClientFactory httpClientFactory,
+ IOptions fastApiOptions,
+ ILogger logger)
+ {
+ _sessions = sessions;
+ _agent = agent;
+ _httpClientFactory = httpClientFactory;
+ _fastApiOptions = fastApiOptions;
+ _logger = logger;
+ }
+
+ /// Run one chat turn with SSE streaming (status, tool_start, tool_end, token, narrative_partial, narrative, error, done).
+ [HttpPost("")]
+ [Produces("text/event-stream")]
+ public async Task PostChat([FromBody] ChatRequest body, CancellationToken cancellationToken)
+ {
+ if (body.SessionId == 0 || body.PropertyId == 0 || string.IsNullOrWhiteSpace(body.Message))
+ {
+ Response.StatusCode = StatusCodes.Status400BadRequest;
+ await Response.WriteAsJsonAsync(new { detail = "sessionId, propertyId, and message required" }, cancellationToken);
+ return;
+ }
+
+ var session = await _sessions.GetSessionAsync(body.SessionId, cancellationToken);
+ if (session is null || session.PropertyId != body.PropertyId)
+ {
+ Response.StatusCode = StatusCodes.Status404NotFound;
+ await Response.WriteAsJsonAsync(new { detail = "session not found" }, cancellationToken);
+ return;
+ }
+
+ await _sessions.AppendMessageAsync(body.SessionId, "user", body.Message.Trim(), cancellationToken: cancellationToken);
+
+ var history = await _sessions.GetMessagesAsync(body.SessionId, cancellationToken: cancellationToken);
+ var agentMessages = ChatHelpers.MessagesForAgentContext(history);
+ var context = new AuditToolContext
+ {
+ PropertyId = (int)body.PropertyId,
+ ReportId = body.ReportId,
+ };
+
+ Response.ContentType = "text/event-stream";
+ Response.Headers.CacheControl = "no-cache";
+
+ var channel = Channel.CreateUnbounded();
+ ChatTurnResult? agentResult = null;
+
+ var agentTask = Task.Run(async () =>
+ {
+ try
+ {
+ agentResult = await _agent.RunTurnAsync(
+ agentMessages,
+ context,
+ evt => channel.Writer.TryWrite(ChatSseSerializer.ToJson(evt)),
+ cancellationToken);
+ }
+ catch (Exception ex)
+ {
+ channel.Writer.TryWrite(ChatSseSerializer.ToJson(new ChatErrorStreamEvent(ex.Message)));
+ }
+ finally
+ {
+ channel.Writer.Complete();
+ }
+ }, cancellationToken);
+
+ await foreach (var item in channel.Reader.ReadAllAsync(cancellationToken))
+ {
+ var eventType = item["type"]?.GetValue() ?? "message";
+ await Response.WriteAsync($"event: {eventType}\n", cancellationToken);
+ await Response.WriteAsync($"data: {item.ToJsonString()}\n\n", cancellationToken);
+ await Response.Body.FlushAsync(cancellationToken);
+ }
+
+ await agentTask;
+
+ var toolResultJson = agentResult is null ? null : ChatPersistenceMapper.ToToolResultJson(agentResult);
+ if (toolResultJson is not null)
+ {
+ try
+ {
+ await _sessions.AppendMessageAsync(
+ body.SessionId,
+ "assistant",
+ content: "",
+ toolResultJson: toolResultJson,
+ cancellationToken: cancellationToken);
+ if (session.Title is "New chat" or "" or null)
+ {
+ var derived = ChatHelpers.DeriveTitle(body.Message)
+ ?? ChatHelpers.DeriveTitle(ChatPersistenceMapper.FirstNarrativeInsight(agentResult!));
+ if (!string.IsNullOrWhiteSpace(derived))
+ {
+ await _sessions.UpdateSessionTitleAsync(body.SessionId, derived, cancellationToken);
+ }
+ }
+ }
+ catch (Exception ex)
+ {
+ _logger.LogWarning(ex, "Failed to persist chat assistant message for session {SessionId}", body.SessionId);
+ }
+ }
+ }
+
+ [HttpGet("sessions")]
+ [ProducesResponseType(StatusCodes.Status200OK)]
+ [ProducesResponseType(StatusCodes.Status400BadRequest)]
+ public async Task ListSessions([FromQuery] long propertyId, CancellationToken cancellationToken)
+ {
+ if (propertyId == 0)
+ {
+ return BadRequest(new { detail = "propertyId required" });
+ }
+
+ var sessions = await _sessions.ListSessionsAsync(propertyId, cancellationToken: cancellationToken);
+ return Ok(new { sessions = sessions.Select(ChatHelpers.FormatSession).ToList() });
+ }
+
+ [HttpPost("sessions")]
+ [ProducesResponseType(StatusCodes.Status200OK)]
+ [ProducesResponseType(StatusCodes.Status400BadRequest)]
+ public async Task CreateSession([FromBody] ChatSessionCreate body, CancellationToken cancellationToken)
+ {
+ if (body.PropertyId == 0)
+ {
+ return BadRequest(new { detail = "propertyId required" });
+ }
+
+ var title = string.IsNullOrWhiteSpace(body.Title) ? "New chat" : body.Title.Trim();
+ var id = await _sessions.CreateSessionAsync(body.PropertyId, title, cancellationToken);
+ return Ok(new { id, propertyId = body.PropertyId, title });
+ }
+
+ [HttpGet("sessions/{sessionId:long}")]
+ [ProducesResponseType(StatusCodes.Status200OK)]
+ [ProducesResponseType(StatusCodes.Status404NotFound)]
+ public async Task GetSession(long sessionId, CancellationToken cancellationToken)
+ {
+ var session = await _sessions.GetSessionAsync(sessionId, cancellationToken);
+ if (session is null)
+ {
+ return NotFound(new { detail = "session not found" });
+ }
+
+ return Ok(new { session = ChatHelpers.FormatSession(session) });
+ }
+
+ [HttpDelete("sessions/{sessionId:long}")]
+ [ProducesResponseType(StatusCodes.Status200OK)]
+ [ProducesResponseType(StatusCodes.Status404NotFound)]
+ public async Task DeleteSession(
+ long sessionId,
+ [FromQuery] long propertyId,
+ CancellationToken cancellationToken)
+ {
+ var session = await _sessions.GetSessionAsync(sessionId, cancellationToken);
+ if (session is null || session.PropertyId != propertyId)
+ {
+ return NotFound(new { detail = "session not found" });
+ }
+
+ var deleted = await _sessions.DeleteSessionAsync(sessionId, cancellationToken);
+ if (!deleted)
+ {
+ return NotFound(new { detail = "session not found" });
+ }
+
+ return Ok(new { ok = true });
+ }
+
+ [HttpGet("sessions/{sessionId:long}/messages")]
+ [ProducesResponseType(StatusCodes.Status200OK)]
+ [ProducesResponseType(StatusCodes.Status404NotFound)]
+ public async Task GetMessages(
+ long sessionId,
+ [FromQuery] long propertyId,
+ CancellationToken cancellationToken)
+ {
+ var session = await _sessions.GetSessionAsync(sessionId, cancellationToken);
+ if (session is null || session.PropertyId != propertyId)
+ {
+ return NotFound(new { detail = "session not found" });
+ }
+
+ var messages = await _sessions.GetMessagesAsync(sessionId, cancellationToken: cancellationToken);
+ return Ok(new { messages = ChatHelpers.FormatMessages(messages) });
+ }
+
+ [HttpGet("artifacts/{artifactId}")]
+ [ProducesResponseType(StatusCodes.Status200OK)]
+ [ProducesResponseType(StatusCodes.Status400BadRequest)]
+ [ProducesResponseType(StatusCodes.Status404NotFound)]
+ public async Task GetArtifact(string artifactId, CancellationToken cancellationToken)
+ {
+ if (!System.Text.RegularExpressions.Regex.IsMatch(artifactId, @"^[a-f0-9\-]{36}$"))
+ {
+ return BadRequest(new { detail = "Invalid artifact id" });
+ }
+
+ var baseUrl = _fastApiOptions.Value.BaseUrl.Trim().TrimEnd('/');
+ var client = _httpClientFactory.CreateClient();
+ using var response = await client.GetAsync(
+ $"{baseUrl}/api/chat/artifacts/{artifactId}",
+ HttpCompletionOption.ResponseHeadersRead,
+ cancellationToken);
+
+ if (response.StatusCode == System.Net.HttpStatusCode.NotFound)
+ {
+ return NotFound(new { detail = "Artifact not found" });
+ }
+
+ if (!response.IsSuccessStatusCode)
+ {
+ return StatusCode((int)response.StatusCode, new { detail = "Artifact fetch failed" });
+ }
+
+ var bytes = await response.Content.ReadAsByteArrayAsync(cancellationToken);
+ var contentType = response.Content.Headers.ContentType?.MediaType ?? "application/octet-stream";
+ var fileResult = File(bytes, contentType);
+ if (response.Content.Headers.ContentDisposition is { } disposition)
+ {
+ fileResult.FileDownloadName = disposition.FileName?.Trim('"');
+ }
+
+ return fileResult;
+ }
+}
diff --git a/services/AiService/src/AiService.Api/Controllers/ContentController.cs b/services/AiService/src/AiService.Api/Controllers/ContentController.cs
new file mode 100644
index 00000000..7da106e1
--- /dev/null
+++ b/services/AiService/src/AiService.Api/Controllers/ContentController.cs
@@ -0,0 +1,101 @@
+using System.Text.Json.Nodes;
+using AiService.Application.Dto;
+using AiService.Application.Services;
+using Microsoft.AspNetCore.Mvc;
+
+namespace AiService.Api.Controllers;
+
+/// Content studio endpoints — POST /api/content/analyze and wizard.
+[ApiController]
+[Route("api/content")]
+[Tags("Content")]
+public sealed class ContentController : ControllerBase
+{
+ private static readonly HashSet ValidWizardSteps = new(StringComparer.Ordinal)
+ {
+ "intents", "content_types", "tones", "titles", "outline", "draft", "research",
+ };
+
+ private readonly ContentAnalyzeService _analyze;
+ private readonly ContentWizardService _wizard;
+
+ public ContentController(ContentAnalyzeService analyze, ContentWizardService wizard)
+ {
+ _analyze = analyze;
+ _wizard = wizard;
+ }
+
+ [HttpPost("analyze")]
+ [ProducesResponseType(StatusCodes.Status200OK)]
+ [ProducesResponseType(StatusCodes.Status400BadRequest)]
+ [ProducesResponseType(StatusCodes.Status500InternalServerError)]
+ public async Task Analyze([FromBody] JsonObject body, CancellationToken cancellationToken)
+ {
+ var keyword = (body["keyword"]?.GetValue() ?? "").Trim();
+ if (string.IsNullOrEmpty(keyword))
+ {
+ return BadRequest(new { detail = "keyword required" });
+ }
+
+ int? propertyId = body["propertyId"]?.GetValue();
+ try
+ {
+ var analysis = await _analyze.AnalyzeAsync(
+ propertyId,
+ keyword,
+ body["bodyHtml"]?.GetValue() ?? "",
+ body["titleTag"]?.GetValue() ?? "",
+ body["metaDescription"]?.GetValue() ?? "",
+ body["landingUrl"]?.GetValue(),
+ body["useAi"]?.GetValue() == true,
+ body.GetRefresh(),
+ body["title"]?.GetValue() ?? "",
+ cancellationToken);
+
+ return Ok(new { analysis });
+ }
+ catch (Exception ex)
+ {
+ return StatusCode(StatusCodes.Status500InternalServerError, new { detail = $"Content analyze failed: {ex.Message}" });
+ }
+ }
+
+ [HttpPost("wizard")]
+ [ProducesResponseType(StatusCodes.Status200OK)]
+ [ProducesResponseType(StatusCodes.Status400BadRequest)]
+ [ProducesResponseType(StatusCodes.Status500InternalServerError)]
+ public async Task Wizard([FromBody] JsonObject body, CancellationToken cancellationToken)
+ {
+ var step = (body["step"]?.GetValue() ?? "").Trim();
+ if (!ValidWizardSteps.Contains(step))
+ {
+ return BadRequest(new { detail = "Invalid wizard step" });
+ }
+
+ var payload = new JsonObject
+ {
+ ["keyword"] = body["keyword"]?.GetValue() ?? "",
+ ["locale"] = body["locale"]?.GetValue() ?? "en-US",
+ ["intent"] = body["intent"]?.GetValue() ?? "",
+ ["contentType"] = body["contentType"]?.GetValue() ?? "",
+ ["tone"] = body["tone"]?.GetValue() ?? "",
+ ["title"] = body["title"]?.GetValue() ?? "",
+ ["outline"] = body["outline"] is JsonArray outline ? outline.DeepClone() : new JsonArray(),
+ };
+
+ try
+ {
+ var result = await _wizard.RunStepAsync(step, payload, cancellationToken);
+ if (result["ok"]?.GetValue() == false)
+ {
+ return BadRequest(new { detail = result["error"]?.GetValue() ?? "Wizard step failed" });
+ }
+
+ return Ok(new { result });
+ }
+ catch (Exception ex)
+ {
+ return StatusCode(StatusCodes.Status500InternalServerError, new { detail = $"Wizard step failed: {ex.Message}" });
+ }
+ }
+}
diff --git a/services/AiService/src/AiService.Api/Controllers/ControllerHelpers.cs b/services/AiService/src/AiService.Api/Controllers/ControllerHelpers.cs
new file mode 100644
index 00000000..da8c2a00
--- /dev/null
+++ b/services/AiService/src/AiService.Api/Controllers/ControllerHelpers.cs
@@ -0,0 +1,118 @@
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Text.RegularExpressions;
+using AiService.Application.Chat;
+using AiService.Domain.Entities;
+
+namespace AiService.Api.Controllers;
+
+internal static class ChatHelpers
+{
+ private static readonly Regex FirstSentenceRe = new(@"^(.{8,80}[.!?])", RegexOptions.Singleline);
+
+ internal static object FormatSession(ChatSession session) => new
+ {
+ id = session.Id,
+ propertyId = session.PropertyId,
+ title = session.Title,
+ createdAt = session.CreatedAt,
+ updatedAt = session.UpdatedAt,
+ };
+
+ internal static List