diff --git a/.coveragerc b/.coveragerc
index d0d0448b..122ffd55 100644
--- a/.coveragerc
+++ b/.coveragerc
@@ -19,6 +19,11 @@ omit =
*/website_profiling/llm_config.py
*/website_profiling/cli.py
*/website_profiling/commands/enrich_cmd.py
+ # FastAPI server — tested via integration tests, not unit tests
+ */website_profiling/api/*
+ */website_profiling/clients/*
+ # Worker process — requires running DB and subprocess for meaningful tests
+ */website_profiling/worker/*
[report]
show_missing = True
diff --git a/.coveragerc.tools b/.coveragerc.tools
index fbb9a0d5..cfcf3e61 100644
--- a/.coveragerc.tools
+++ b/.coveragerc.tools
@@ -1,5 +1,7 @@
[run]
-source = website_profiling.tools
+source =
+ website_profiling.tools
+ website_profiling.clients
omit =
*/website_profiling/tools/keywords.py
*/website_profiling/tools/plot.py
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index cf5abd8c..81c94041 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -43,7 +43,7 @@ jobs:
--cov-report=term-missing --cov-fail-under=100 -q -o addopts=
- name: Pytest (tools coverage gate)
run: |
- pytest tests/tools/ \
+ pytest tests/tools/ tests/clients/ \
--cov=website_profiling.tools --cov-config=.coveragerc.tools \
--cov-report=term-missing --cov-fail-under=100 -q -o addopts=
- name: CLI smoke
@@ -88,3 +88,13 @@ jobs:
run: npm run lint
- name: Test
run: npm test
+
+ files:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-dotnet@v4
+ with:
+ dotnet-version: '10.0.x'
+ - name: Test FileService
+ run: dotnet test services/FileService/FileService.slnx
diff --git a/.gitignore b/.gitignore
index b19c6956..5ea03af9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,4 @@
-# Project (root) only. Python: src/.gitignore. Next.js: web/.gitignore
+# Project (root) only. Python: src/.gitignore. Next.js: web/.gitignore. .NET: services/FileService/
# Next.js UI: generated pipeline configs from the runner modal (repo root; must match Python cwd for paths)
.website-profiling-ui-*.txt
@@ -32,4 +32,13 @@ pipeline-config.txt
.agents/
skills-lock.json
crawl_results.csv
-commit.*
\ No newline at end of file
+commit.*
+
+# .NET FileService — build output and IDE artifacts
+services/FileService/**/bin/
+services/FileService/**/obj/
+services/FileService/.vs/
+services/FileService/**/*.user
+services/FileService/**/*.suo
+services/FileService/**/TestResults/
+services/FileService/**/*.DotSettings.user
\ No newline at end of file
diff --git a/AGENT.md b/AGENT.md
index 62427d68..e51865f6 100644
--- a/AGENT.md
+++ b/AGENT.md
@@ -11,10 +11,11 @@ Developer reference for agents and contributors. User-facing overview: [README.m
**Key paths**
- `src/website_profiling/` -- `cli.py`, `config.py`, `crawl/`, `db/storage.py`, `lighthouse/`, `reporting/`, `analysis/`, `llm/`, `tools/`
+- `services/FileService/` -- .NET PDF + Excel workbook export (HTTP-only; see [README](services/FileService/README.md))
- `web/app/` -- routes; `web/src/` -- React; pipeline: `PipelineRunnerFab`, `server/pipelineJobs.ts`, `server/pipelineConfig.ts`, `server/llmConfig.ts`, `server/db.ts`
- `alembic/` -- schema migrations
-**Local dev:** `./local-run` (Postgres in Docker `wp-pg`, Next.js on host; default `DATABASE_URL`: `postgres://postgres:dev@127.0.0.1:5432/website_profiling`). See `scripts/local-run.sh`. **Local tests:** `./local-test` runs **three** Python coverage gates (core 100%, reporting 100%, tools 100%) plus web checks — mirrors CI **python** and **web** jobs; Docker CI is separate (see `.github/workflows/ci.yml`). `./local-test browser` for `@pytest.mark.browser` integration tests — see `scripts/local-test.sh`. Mocked browser unit tests: `tests/test_browser_fetcher_unit.py`.
+**Local dev:** `./local-run` (Postgres in Docker `wp-pg`, FileService on `:8080`, Next.js on host; default `DATABASE_URL`: `postgres://postgres:dev@127.0.0.1:5432/website_profiling`). See `scripts/local-run.sh`. **Local tests:** `./local-test` runs **three** Python coverage gates (core 100%, reporting 100%, tools 100%) plus web checks — mirrors CI **python** and **web** jobs; Docker CI is separate (see `.github/workflows/ci.yml`). `./local-test browser` for `@pytest.mark.browser` integration tests — see `scripts/local-test.sh`. Mocked browser unit tests: `tests/test_browser_fetcher_unit.py`.
**JavaScript crawl (optional):** Config keys `crawl_render_mode` (`static` | `javascript` | `auto`) and `crawl_js_*` in pipeline config / `pipelineConfigSchema.ts`. JS/auto crawls can capture browser console errors and uncaught exceptions (`crawl_js_capture_console`, stored under `page_analysis.browser`). **Auto mode** uses static-first fetch, pre-parse SPA heuristics (`needs_js_render`), then post-parse low-outlink fallback (`needs_js_render_after_parse`) in `crawler.py`. **Preflight:** `GET /api/crawl/browser-status` (localhost) spawns Python `browser_status()`; Run audit settings/run validation calls it when render mode is `javascript` or `auto`. Browser deps: Playwright from `requirements.txt` (installed by `./local-run setup` and `./local-test`). Runtime needs Chromium on `PATH` or `CHROME_PATH` (Docker sets `CHROME_PATH=/usr/bin/chromium`). Integration tests: `@pytest.mark.browser` — excluded by default in `pytest.ini`; Docker CI runs `tests/test_crawl_fetchers.py` and `tests/test_crawler_browser_e2e.py -m browser`; locally `./local-test browser`.
@@ -31,7 +32,7 @@ Developer reference for agents and contributors. User-facing overview: [README.m
- **AI Chat UI:** `/chat` — property-scoped chat with saved sessions (`chat_sessions`, `chat_messages`; migration `012_chat_sessions`).
- **Job store:** PostgreSQL `pipeline_jobs` when `DATABASE_URL` is set (`pipelineJobsDb.ts` — status, timestamps, truncated logs). In-memory map in `pipelineJobs.ts` holds live log tail and child process handles; stale rows reconciled via `PIPELINE_JOB_STALE_HOURS`.
- **Schema head:** `015_crawl_page_html` (recent: `013` link_edges/discovery, `014` job log truncation, `015` per-URL HTML storage).
-- **Docker:** `Dockerfile` + `docker-compose.yml` (postgres + web); **`docker-compose.prod.yml`** (production + remote MCP on `:8000`); **`docker-compose.pull.yml`** for pre-built images (`WEB_IMAGE`); **`LIGHTHOUSE_CHROME_FLAGS`**
+- **Docker:** `Dockerfile` + `docker-compose.yml` (postgres + web + FileService); **`docker-compose.prod.yml`** (production + remote MCP on `:8000`); **`docker-compose.pull.yml`** for pre-built images (`WEB_IMAGE`); **`LIGHTHOUSE_CHROME_FLAGS`**
**Where to edit**
@@ -39,6 +40,7 @@ Developer reference for agents and contributors. User-facing overview: [README.m
|------|--------|
| Crawl | `crawl/crawler.py`, `crawl/fetchers/` |
| Report | `reporting/builder.py`, `reporting/categories.py` |
+| PDF / workbook export | `services/FileService/` (rendering); Next.js proxies in `web/src/server/proxyToFileService.ts` |
| DB schema | `alembic/versions/` |
| Local analysis | `analysis/local.py`, `requirements.txt` |
| AI insights (LLM) | `llm/enrich.py`, `llm/agent.py`, `llm_config.py`, `requirements.txt` |
@@ -84,7 +86,7 @@ The web UI uses **both** Chart.js and D3.js. Pick the library that fits each cha
- Keep chart-library types out of data-prep: use neutral shapes (`BarChartData`, `DualSeriesChartData` in `web/src/lib/viz/types.ts` and `web/src/lib/compareChartData.ts`); convert at the render layer via `web/src/lib/viz/adapters.ts` when needed.
- Migrate page-by-page when D3 is the better fit; do not remove `chart.js` from `package.json` until all consumers are migrated.
-**Company standards:** UI copy in `web/src/strings.json` (Site Audit, Properties, Run audit). Data provenance on `report_meta` in report payload. Docs: `docs/COMPANY_STANDARDS.md`, `docs/GLOSSARY.md`. Migration `003_company_standards` (properties, pipeline_jobs, audit_log). Durable jobs in `web/src/server/pipelineJobsDb.ts`. Export: `GET /api/report/export`, `src/website_profiling/tools/export_audit.py`.
+**Company standards:** UI copy in `web/src/strings.json` (Site Audit, Properties, Run audit). Data provenance on `report_meta` in report payload. Docs: `docs/COMPANY_STANDARDS.md`, `docs/GLOSSARY.md`. Migration `003_company_standards` (properties, pipeline_jobs, audit_log). Durable jobs in `web/src/server/pipelineJobsDb.ts`. **Export:** PDF/workbook via FileService (`FILE_SERVICE_URL` on web/MCP; `REPORT_API_URL` on FileService); CSV/JSON via `GET /api/report/export` and `src/website_profiling/tools/export_audit.py`.
**Common footguns (check before finishing web or DB work)**
diff --git a/AGENTS.md b/AGENTS.md
index 878fe63e..be63f52f 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -11,6 +11,7 @@ This file is the canonical entry point for agents. For full detail see [AGENT.md
- `src/website_profiling/` — core Python package
- `cli.py`, `config.py`, `crawl/`, `db/`, `reporting/`, `analysis/`, `llm/`, `tools/`
- `web/` — Next.js frontend
+- `services/FileService/` — .NET PDF + Excel workbook export (port 8080). HTTP-only via `REPORT_API_URL`; no Postgres. Profiles: `executive|standard|full|premium`. Details: [services/FileService/README.md](services/FileService/README.md). Env: `FILE_SERVICE_URL` (Next.js/MCP), `REPORT_API_URL` (FileService).
- `alembic/` — DB migrations
- `docs/` — documentation index
- `tests/` — pytest suite
@@ -18,7 +19,7 @@ This file is the canonical entry point for agents. For full detail see [AGENT.md
**Run / dev**
```bash
-./local-run # Start Postgres (Docker) + Next.js
+./local-run # Start Postgres + FileService + Next.js
./local-test # Run all three coverage gates
python -m src # Run audit pipeline
python -m website_profiling.mcp # Start MCP server (stdio)
diff --git a/Dockerfile b/Dockerfile
index a8222a01..94d185bd 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,5 +1,5 @@
# syntax=docker/dockerfile:1
-# WebsiteProfiling: Next.js web UI + Python pipeline (spawned from /api/run).
+# WebsiteProfiling: Next.js UI + FastAPI (port 8001) + Python worker + pipeline.
# Build from repository root: docker build -t website-profiling .
# BuildKit cache mounts (default in Docker Desktop) reuse pip/npm downloads across rebuilds.
diff --git a/README.md b/README.md
index 15fff6ec..45b47f6f 100644
--- a/README.md
+++ b/README.md
@@ -19,6 +19,7 @@
+ 
@@ -84,7 +85,7 @@ Audit → Report → MCP → Fix → Review → (repeat)
| Step | What you do | In Site Audit |
|------|-------------|---------------|
| **Audit** | Crawl and score the site | Pipeline (`python -m src`), Lighthouse, on-page checks |
-| **Report** | Export and prioritize fixes | PDF/HTML/CSV exports, issue board, fix roadmap |
+| **Report** | Export and prioritize fixes | PDF, Excel workbook, CSV, and JSON exports; issue board; fix roadmap |
| **MCP** | Pull audit context into your IDE | `python -m website_profiling.mcp` — read-only tools for Cursor / Claude Desktop |
| **Fix** | Ship changes in your codebase | Your PR workflow (MCP does not write to the site) |
| **Review** | Prove improvement | Compare runs, category deltas, GSC metric changes |
@@ -159,12 +160,13 @@ WebsiteProfiling/
│ ├── src/views/ # Report views (overview, links, issues, …)
│ ├── src/server/ # Server-side DB, pipeline jobs, config I/O
│ └── public/ # Static assets (logo, favicon)
+├── services/FileService/ # .NET PDF + Excel workbook export (port 8080)
├── alembic/versions/ # PostgreSQL schema migrations
├── tests/ # pytest suite + fixtures
├── docs/ # Glossary, MCP, ops, brand assets
├── scripts/ # local-run.sh, local-test.sh helpers
├── .github/workflows/ # CI (Python + web + browser crawl)
-├── docker-compose.yml # Dev stack (Postgres + web)
+├── docker-compose.yml # Dev stack (Postgres + web + FileService)
├── docker-compose.prod.yml # Production stack (requires AUTH_SECRET)
├── docker-compose.pull.yml # Pre-built WEB_IMAGE
├── Dockerfile # Production image
@@ -178,6 +180,7 @@ WebsiteProfiling/
| Path | Purpose |
| ------------------------------------- | ------------------------------------------------------------------------------ |
| `src/website_profiling/` | Crawl, analyze, report, Lighthouse, integrations, AI — run via `python -m src` |
+| `services/FileService/` | PDF and Excel workbook export — see [services/FileService/README.md](services/FileService/README.md) |
| `web/app/api/` | REST APIs: report data, pipeline runs, chat (SSE), Google/Bing sync |
| `web/src/lib/pipelineConfigSchema.ts` | Audit settings schema (UI ↔ PostgreSQL) |
| `alembic/versions/` | Database migrations — run `./local-run migrate` |
@@ -198,7 +201,7 @@ Build and run from source:
docker compose up --build
```
-Open [http://localhost:3000/home](http://localhost:3000/home).
+Open [http://localhost:3000/home](http://localhost:3000/home). PDF and workbook exports require the **FileService** container (`files`, port 8080).
Production deployment: `docker-compose.prod.yml` — set `POSTGRES_USER`, `POSTGRES_PASSWORD`, and `AUTH_SECRET`. Pre-built images: `docker-compose.pull.yml` (`WEB_IMAGE`).
@@ -206,7 +209,7 @@ Production deployment: `docker-compose.prod.yml` — set `POSTGRES_USER`, `POSTG
```bash
./local-run setup # First time: Postgres, Python venv, migrations, npm deps
-./local-run # Start DB + Next.js dev server → http://localhost:3000/home
+./local-run # Start DB + FileService + Next.js → http://localhost:3000/home
./local-run db # Postgres only (no app)
./local-run migrate # Apply Alembic migrations only
./local-run stop # Stop Postgres container
diff --git a/alembic/versions/025_pipeline_job_queue.py b/alembic/versions/025_pipeline_job_queue.py
new file mode 100644
index 00000000..8a20f20f
--- /dev/null
+++ b/alembic/versions/025_pipeline_job_queue.py
@@ -0,0 +1,35 @@
+"""Add pipeline job queue columns for the Python worker.
+
+Adds: command, cancel_requested, pause_requested, worker_pid, status='pending'.
+
+Revision ID: 025_pipeline_job_queue
+Revises: 014_pipeline_log_truncated
+"""
+from __future__ import annotations
+
+from alembic import op
+
+revision = "025_pipeline_job_queue"
+down_revision = "024_app_settings"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ op.execute("""
+ ALTER TABLE pipeline_jobs
+ ADD COLUMN IF NOT EXISTS command TEXT,
+ ADD COLUMN IF NOT EXISTS cancel_requested BOOLEAN NOT NULL DEFAULT false,
+ ADD COLUMN IF NOT EXISTS pause_requested BOOLEAN NOT NULL DEFAULT false,
+ ADD COLUMN IF NOT EXISTS worker_pid INTEGER;
+ """)
+
+
+def downgrade() -> None:
+ op.execute("""
+ ALTER TABLE pipeline_jobs
+ DROP COLUMN IF EXISTS worker_pid,
+ DROP COLUMN IF EXISTS pause_requested,
+ DROP COLUMN IF EXISTS cancel_requested,
+ DROP COLUMN IF EXISTS command;
+ """)
diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml
index 0ba7a67b..e1ccf357 100644
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -23,6 +23,7 @@ services:
condition: service_healthy
ports:
- '${WEB_PORT:-3000}:3000'
+ - '${FASTAPI_PORT:-8001}:8001'
environment:
WEBSITE_PROFILING_ROOT: /app
DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-website_profiling}
@@ -30,6 +31,13 @@ services:
AUTH_SECRET: ${AUTH_SECRET:?set AUTH_SECRET}
AUTH_PASSWORD: ${AUTH_PASSWORD:-}
NODE_ENV: production
+ FASTAPI_URL: http://127.0.0.1:8001
+ FASTAPI_ALLOWED_ORIGINS: ${FASTAPI_ALLOWED_ORIGINS:-http://localhost:3000}
+ FILE_SERVICE_URL: http://files:8080
+ PYTHON: /opt/venv/bin/python
+ CHROME_PATH: /usr/bin/chromium
+ LIGHTHOUSE_PATH: /usr/local/bin/lighthouse
+ LIGHTHOUSE_CHROME_FLAGS: --headless --no-sandbox --disable-dev-shm-usage --disable-gpu
volumes:
- profiling-data:/data
healthcheck:
@@ -37,6 +45,7 @@ services:
interval: 30s
timeout: 5s
retries: 3
+ start_period: 30s
worker:
build:
@@ -45,7 +54,7 @@ services:
depends_on:
postgres:
condition: service_healthy
- command: ['python', '-m', 'src', 'report']
+ command: ['/opt/venv/bin/python', '-m', 'website_profiling.worker']
environment:
WEBSITE_PROFILING_ROOT: /app
DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-website_profiling}
@@ -55,6 +64,17 @@ services:
profiles:
- worker
+ files:
+ build:
+ context: ./services/FileService
+ ports:
+ - '${FILE_SERVICE_PORT:-8080}:8080'
+ environment:
+ REPORT_API_URL: http://web:8001
+ depends_on:
+ web:
+ condition: service_started
+
mcp:
build:
context: .
@@ -62,10 +82,13 @@ services:
depends_on:
postgres:
condition: service_healthy
+ files:
+ condition: service_started
command: ['python', '-m', 'website_profiling.mcp.http']
environment:
WEBSITE_PROFILING_ROOT: /app
DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-website_profiling}
+ FILE_SERVICE_URL: http://files:8080
WP_MCP_HTTP_HOST: 0.0.0.0
WP_MCP_HTTP_PORT: 8000
WP_MCP_TOKEN: ${WP_MCP_TOKEN:?set WP_MCP_TOKEN}
diff --git a/docker-compose.pull.yml b/docker-compose.pull.yml
index b89e0767..67d072ca 100644
--- a/docker-compose.pull.yml
+++ b/docker-compose.pull.yml
@@ -33,6 +33,8 @@ services:
CHROME_PATH: /usr/bin/chromium
LIGHTHOUSE_PATH: /usr/local/bin/lighthouse
LIGHTHOUSE_CHROME_FLAGS: --headless --no-sandbox --disable-dev-shm-usage --disable-gpu
+ FASTAPI_URL: http://127.0.0.1:8001
+ FILE_SERVICE_URL: http://files:8080
volumes:
- profiling-data:/data
healthcheck:
@@ -42,6 +44,17 @@ services:
retries: 3
start_period: 15s
+ files:
+ build:
+ context: ./services/FileService
+ ports:
+ - "8080:8080"
+ environment:
+ REPORT_API_URL: http://web:8001
+ depends_on:
+ web:
+ condition: service_started
+
volumes:
pg-data:
profiling-data:
diff --git a/docker-compose.yml b/docker-compose.yml
index b4ecaa40..02427c6c 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -23,6 +23,7 @@ services:
condition: service_healthy
ports:
- "3000:3000"
+ - "8001:8001"
environment:
WEBSITE_PROFILING_ROOT: /app
DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling
@@ -32,6 +33,9 @@ services:
CHROME_PATH: /usr/bin/chromium
LIGHTHOUSE_PATH: /usr/local/bin/lighthouse
LIGHTHOUSE_CHROME_FLAGS: --headless --no-sandbox --disable-dev-shm-usage --disable-gpu
+ FASTAPI_URL: http://127.0.0.1:8001
+ FASTAPI_ALLOWED_ORIGINS: "http://localhost:3000"
+ FILE_SERVICE_URL: http://files:8080
volumes:
- profiling-data:/data
healthcheck:
@@ -41,6 +45,17 @@ services:
retries: 3
start_period: 15s
+ files:
+ build:
+ context: ./services/FileService
+ ports:
+ - "8080:8080"
+ environment:
+ REPORT_API_URL: http://web:8001
+ depends_on:
+ web:
+ condition: service_started
+
# Optional remote MCP (Streamable HTTP). Uncomment and set WP_MCP_TOKEN / WP_MCP_ALLOWED_HOSTS.
# mcp:
# build:
diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh
index 43f4b1e3..d5ec568b 100644
--- a/docker-entrypoint.sh
+++ b/docker-entrypoint.sh
@@ -63,4 +63,48 @@ sys.exit(1)
PY
/opt/venv/bin/alembic upgrade head
-cd /app/web && exec npm run start -- -H 0.0.0.0 -p 3000
+
+WORKER_PID=""
+UVICORN_PID=""
+NPM_PID=""
+
+cleanup() {
+ [ -n "$WORKER_PID" ] && kill "$WORKER_PID" 2>/dev/null || true
+ [ -n "$UVICORN_PID" ] && kill "$UVICORN_PID" 2>/dev/null || true
+ [ -n "$NPM_PID" ] && kill "$NPM_PID" 2>/dev/null || true
+}
+trap cleanup TERM INT
+
+/opt/venv/bin/python -m website_profiling.worker &
+WORKER_PID=$!
+
+/opt/venv/bin/uvicorn website_profiling.api.main:app \
+ --host 0.0.0.0 --port 8001 --workers 1 &
+UVICORN_PID=$!
+
+# Wait for FastAPI to be ready before starting Next.js (max ~15s)
+i=0
+while [ "$i" -lt 30 ]; do
+ if node -e "require('http').get('http://127.0.0.1:8001/api/health',r=>process.exit(r.statusCode===200?0:1)).on('error',()=>process.exit(1))" 2>/dev/null; then
+ echo "FastAPI ready (attempt $((i + 1))/30)" >&2
+ break
+ fi
+ sleep 0.5
+ i=$((i + 1))
+done
+if [ "$i" -eq 30 ]; then
+ echo "WARNING: FastAPI did not respond to /api/health after 15s — continuing anyway" >&2
+fi
+
+cd /app/web
+npm run start -- -H 0.0.0.0 -p 3000 &
+NPM_PID=$!
+
+# Monitor critical processes — exit the container if either npm or uvicorn dies.
+# A dead worker does not break the UI so it is intentionally excluded.
+while kill -0 "$NPM_PID" 2>/dev/null && kill -0 "$UVICORN_PID" 2>/dev/null; do
+ sleep 5
+done
+echo "Critical process (npm or uvicorn) exited — shutting down container" >&2
+cleanup
+exit 1
diff --git a/docs/COMPANY_STANDARDS.md b/docs/COMPANY_STANDARDS.md
index da25ca84..867b9c55 100644
--- a/docs/COMPANY_STANDARDS.md
+++ b/docs/COMPANY_STANDARDS.md
@@ -70,7 +70,7 @@ All metrics displayed in the UI or exports must align with one of the following
|---------|------------|
| **Property** | A client site grouped by canonical domain, with optional GSC/GA4 binding |
| **Audit run** | A stored report snapshot comprising crawl, analysis, and optional Lighthouse or Google data |
-| **Export** | PDF, CSV, or HTML deliverable including a data source legend |
+| **Export** | PDF or Excel workbook (FileService), CSV or JSON (Python), including a data source legend |
Category titles and issue copy in Python use agency vocabulary (`src/website_profiling/reporting/terminology.py`). See [GLOSSARY.md](GLOSSARY.md) for UI term mappings.
diff --git a/docs/GLOSSARY.md b/docs/GLOSSARY.md
index 4d69a9d7..f6d11f2d 100644
--- a/docs/GLOSSARY.md
+++ b/docs/GLOSSARY.md
@@ -52,7 +52,7 @@ This glossary maps agency-facing UI terms to internal keys, database tables, and
| AI Chat | `/chat`, `/api/chat`, `chat_sessions` | LLM + read-only audit tools | Conversational audit queries |
| MCP tools | `python -m website_profiling.mcp` | Same `audit_tools` as chat | IDE integration — see [MCP.md](MCP.md) |
| Read-only session | `AUTH_DEFAULT_ROLE=client-readonly` or `viewer`; `/api/auth/session` returns role and mutation flags | Session cookie | `client-readonly`: view + chat; `viewer`: view only (no chat) |
-| Export executive summary | `export_audit_html/pdf/csv`, Export view | Report payload + optional AI | Client deliverable |
+| Export executive summary | Export view; MCP `export_audit_report` (pdf/csv/json); workbook via Export view or FileService | Report payload + optional AI | Client deliverable |
| ads.txt / security.txt | `site_level`, `get_ads_txt_status`, `get_security_txt_status` | Root file fetch at report build | Publisher / contact file hygiene |
| Subdomain inventory | `subdomains`, `list_subdomains`, `/subdomains` view | Crawl + GSC + optional crt.sh | Host footprint vs crawl scope |
| Contact intelligence | `contact_intelligence`, `get_contact_intelligence`, `/contacts` view | Crawl schema/mailto + security.txt + RDAP org | Business identity consistency |
diff --git a/docs/MCP.md b/docs/MCP.md
index 1c809ac7..7a41f19d 100644
--- a/docs/MCP.md
+++ b/docs/MCP.md
@@ -216,7 +216,14 @@ Export tools write artifact files with a 24-hour TTL; in-app chat renders downlo
`export_audit_report`, `export_compare_csv`, `export_list_as_csv`, `export_sitemap_xml`, `validate_rich_results`, `list_export_formats`
-Full audit exports use the same generators as the Export view. PDF export requires `reportlab`.
+| Deliverable | Generator | Env / notes |
+|-------------|-----------|-------------|
+| PDF | **FileService** (`GET /v1/reports/{id}/pdf`) | `FILE_SERVICE_URL` on MCP/web (default `http://127.0.0.1:8080`); FileService must be running |
+| Excel workbook | **FileService** (`GET /v1/reports/{id}/workbook`) | Same as PDF |
+| CSV / JSON audit export | Python (`export_audit_report`, `GET /api/report/export`) | Reads Postgres via report payload — no FileService required |
+| Compare / list CSV | Python | `export_compare_csv`, `export_list_as_csv` |
+
+FileService fetches report JSON over HTTP only (`REPORT_API_URL` → `/api/report/payload`, `/api/report/meta`, `/api/app-settings`). See [services/FileService/README.md](../services/FileService/README.md).
### Image audit
@@ -368,7 +375,7 @@ The following capabilities are planned but not yet available:
| Security | "Show security finding changes since report 38" |
| Links | "Which pages link to broken URLs?" |
| Content | "Generate a content brief for keyword X" |
-| Export | "Download the audit as PDF" |
+| Export | "Download the audit as PDF" or "Export the crawl workbook as Excel" |
| Compare | "Compare report 38 to the current audit and give me a CSV diff" |
| Client report | "Build a client report with executive summary, category scores, and top critical issues as PDF" |
| Images | "Which images are largest and unoptimized?" |
diff --git a/docs/OPS.md b/docs/OPS.md
index e26c5549..dfa2a14e 100644
--- a/docs/OPS.md
+++ b/docs/OPS.md
@@ -259,6 +259,17 @@ docker compose -f docker-compose.pull.yml up # pre-built WEB_IMAGE
Do not run the application container in isolation with `docker run` unless you provide a reachable `DATABASE_URL`.
+### FileService (PDF and workbook export)
+
+The `files` service (port **8080**) renders audit PDFs and Excel workbooks. It reads report data over HTTP from the `web` service — no Postgres connection.
+
+| Variable | Service | Purpose |
+|----------|---------|---------|
+| `FILE_SERVICE_URL` | `web`, MCP | Where clients call FileService (default `http://files:8080` in Compose) |
+| `REPORT_API_URL` | `files` | Report API base URL (Compose: `http://web:8001`) |
+
+PDF or workbook downloads fail if `files` is not running. See [services/FileService/README.md](../services/FileService/README.md).
+
---
## Running tests
diff --git a/docs/README.md b/docs/README.md
index 24dcbe1c..132a6705 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -15,6 +15,7 @@ This directory contains product, integration, and operations documentation for *
| [COMPANY_STANDARDS.md](COMPANY_STANDARDS.md) | Agencies / operators | Data classification, crawl scope, security policy |
| [MCP.md](MCP.md) | Integrators | Model Context Protocol server configuration and tool reference |
| [OPS.md](OPS.md) | Operators | Scheduled audits, alerts, migrations, production notes |
+| [services/FileService/README.md](../services/FileService/README.md) | Developers / operators | PDF and Excel workbook export service |
---
diff --git a/docs/assets/readme-banner.png b/docs/assets/readme-banner.png
index c0687499..6c77d150 100644
Binary files a/docs/assets/readme-banner.png and b/docs/assets/readme-banner.png differ
diff --git a/requirements.txt b/requirements.txt
index c7bc3f69..aa4ab71b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -32,9 +32,8 @@ psycopg[binary,pool]==3.3.4
sqlalchemy==2.0.50
alembic==1.18.4
-# Audit export (PDF)
-reportlab==4.5.1
+# Tabular audit exports (CSV/JSON) — PDF and workbook via FileService (.NET)
# JavaScript rendering crawl (headless Chromium via Playwright)
playwright==1.60.0
@@ -55,6 +54,7 @@ tiktoken==0.13.0
mcp>=1.19,<2
uvicorn>=0.30
starlette>=0.38
+fastapi>=0.115
# SQL query validation (read-only chat tool)
sqlglot==30.11.0
diff --git a/scripts/generate_openapi.py b/scripts/generate_openapi.py
new file mode 100644
index 00000000..ba11730a
--- /dev/null
+++ b/scripts/generate_openapi.py
@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+"""Write web/openapi.json from the FastAPI app (no running server required)."""
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT / "src"))
+
+from fastapi.openapi.utils import get_openapi # noqa: E402
+
+from website_profiling.api.main import app # noqa: E402
+
+OUT = ROOT / "web" / "openapi.json"
+
+
+def main() -> None:
+ schema = get_openapi(
+ title=app.title,
+ version=app.version,
+ routes=app.routes,
+ )
+ OUT.write_text(json.dumps(schema, indent=2) + "\n", encoding="utf-8")
+ print(f"Wrote {OUT}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/scripts/local-prod.sh b/scripts/local-prod.sh
index 61cd1ab9..37892ad0 100755
--- a/scripts/local-prod.sh
+++ b/scripts/local-prod.sh
@@ -68,9 +68,34 @@ cmd_start() {
log "DATA_DIR=$DATA_DIR"
log "PYTHON=$PYTHON"
log "NODE_ENV=$NODE_ENV"
- cd "$WEB"
+ cd "$ROOT"
export DATABASE_URL DATA_DIR PYTHON WEBSITE_PROFILING_ROOT PYTHONPATH NODE_ENV
- exec npm run start
+
+ WORKER_PID=""
+ UVICORN_PID=""
+ NPM_PID=""
+
+ cleanup_prod() {
+ [ -n "$WORKER_PID" ] && kill "$WORKER_PID" 2>/dev/null || true
+ [ -n "$UVICORN_PID" ] && kill "$UVICORN_PID" 2>/dev/null || true
+ [ -n "$NPM_PID" ] && kill "$NPM_PID" 2>/dev/null || true
+ }
+ trap cleanup_prod INT TERM EXIT
+
+ log "Starting pipeline worker"
+ "$ROOT/.venv/bin/python" -m website_profiling.worker &
+ WORKER_PID=$!
+
+ log "Starting FastAPI on port 8001"
+ export FASTAPI_URL="http://127.0.0.1:8001"
+ "$ROOT/.venv/bin/uvicorn" website_profiling.api.main:app \
+ --host 0.0.0.0 --port 8001 --workers 1 &
+ UVICORN_PID=$!
+
+ cd "$WEB"
+ npm run start -- -H 0.0.0.0 -p 3000 &
+ NPM_PID=$!
+ wait $NPM_PID
}
cmd_help() {
diff --git a/scripts/local-run.sh b/scripts/local-run.sh
index 4f84fea5..6d48b0f0 100755
--- a/scripts/local-run.sh
+++ b/scripts/local-run.sh
@@ -32,6 +32,19 @@ log() { printf '\033[1;36m→\033[0m %s\n' "$*"; }
warn() { printf '\033[1;33m!\033[0m %s\n' "$*" >&2; }
die() { printf '\033[1;31m✗\033[0m %s\n' "$*" >&2; exit 1; }
+# Kill any process still listening on a TCP port (stale dev servers after Ctrl+C).
+free_port() {
+ local port="$1"
+ local pids
+ pids="$(lsof -nP -tiTCP:"$port" -sTCP:LISTEN 2>/dev/null || true)"
+ if [[ -n "$pids" ]]; then
+ warn "Stopping stale listener on port $port (PID(s): ${pids//$'\n'/ })"
+ # shellcheck disable=SC2086
+ kill $pids 2>/dev/null || true
+ sleep 0.3
+ fi
+}
+
need_cmd() {
command -v "$1" >/dev/null 2>&1 || die "Missing required command: $1"
}
@@ -134,13 +147,54 @@ cmd_start() {
log "Ensuring migrations are up to date"
"$VENV/bin/alembic" upgrade head
cmd_web_deps
+ cd "$ROOT"
+ export DATABASE_URL DATA_DIR PYTHON WEBSITE_PROFILING_ROOT PYTHONPATH
+
+ WORKER_PID=""
+ UVICORN_PID=""
+ FILE_SERVICE_PID=""
+
+ cleanup_local() {
+ [ -n "$WORKER_PID" ] && kill "$WORKER_PID" 2>/dev/null || true
+ [ -n "$UVICORN_PID" ] && kill "$UVICORN_PID" 2>/dev/null || true
+ [ -n "$FILE_SERVICE_PID" ] && kill "$FILE_SERVICE_PID" 2>/dev/null || true
+ }
+ trap cleanup_local INT TERM EXIT
+
+ if command -v dotnet >/dev/null 2>&1; then
+ free_port 8080
+ log "Starting FileService on port 8080"
+ export REPORT_API_URL="http://127.0.0.1:8001"
+ (cd "$ROOT/services/FileService" && \
+ ASPNETCORE_URLS="http://127.0.0.1:8080" \
+ ASPNETCORE_ENVIRONMENT=Development \
+ dotnet run --project src/FileService.Api --no-launch-profile) &
+ FILE_SERVICE_PID=$!
+ else
+ warn "dotnet not found — PDF export requires FileService (see services/FileService/README.md)"
+ fi
+
+ log "Starting pipeline worker"
+ "$VENV/bin/python" -m website_profiling.worker &
+ WORKER_PID=$!
+
+ free_port 8001
+ log "Starting FastAPI on port 8001"
+ export FASTAPI_URL="http://127.0.0.1:8001"
+ export FASTAPI_ALLOWED_ORIGINS="http://localhost:3000"
+ "$VENV/bin/uvicorn" website_profiling.api.main:app \
+ --host 0.0.0.0 --port 8001 --workers 1 &
+ UVICORN_PID=$!
+
log "Starting Next.js dev server (Ctrl+C to stop)"
log "DATABASE_URL=$DATABASE_URL"
log "DATA_DIR=$DATA_DIR"
log "PYTHON=$PYTHON"
+ log "FILE_SERVICE_URL=${FILE_SERVICE_URL:-http://127.0.0.1:8080}"
+ export FILE_SERVICE_URL="${FILE_SERVICE_URL:-http://127.0.0.1:8080}"
cd "$WEB"
- export DATABASE_URL DATA_DIR PYTHON WEBSITE_PROFILING_ROOT PYTHONPATH
- exec npm run dev
+ # Do not exec — keep this shell alive so the trap kills FileService/worker/uvicorn on Ctrl+C.
+ npm run dev
}
cmd_stop() {
diff --git a/scripts/local-test.ps1 b/scripts/local-test.ps1
index 7d2d782a..2c8563af 100644
--- a/scripts/local-test.ps1
+++ b/scripts/local-test.ps1
@@ -240,6 +240,7 @@ function Invoke-PytestTools {
Write-Log "Pytest (tools coverage gate, 100%)"
& $VENV_PYTEST `
tests/tools/ `
+ tests/clients/ `
--cov=website_profiling.tools `
--cov-config=.coveragerc.tools `
--cov-report=term-missing `
diff --git a/scripts/local-test.sh b/scripts/local-test.sh
index 174d5725..b16595d9 100755
--- a/scripts/local-test.sh
+++ b/scripts/local-test.sh
@@ -129,6 +129,7 @@ run_pytest_tools() {
log "Pytest (tools coverage gate, 100%)"
"$VENV/bin/pytest" \
tests/tools/ \
+ tests/clients/ \
--cov=website_profiling.tools \
--cov-config=.coveragerc.tools \
--cov-report=term-missing \
diff --git a/services/FileService/.dockerignore b/services/FileService/.dockerignore
new file mode 100644
index 00000000..8682023a
--- /dev/null
+++ b/services/FileService/.dockerignore
@@ -0,0 +1,3 @@
+**/bin/
+**/obj/
+**/.vs/
diff --git a/services/FileService/Dockerfile b/services/FileService/Dockerfile
new file mode 100644
index 00000000..6783a2c1
--- /dev/null
+++ b/services/FileService/Dockerfile
@@ -0,0 +1,17 @@
+FROM mcr.microsoft.com/dotnet/sdk:10.0 AS build
+WORKDIR /src
+COPY FileService.slnx ./
+COPY src/FileService.Domain/FileService.Domain.csproj src/FileService.Domain/
+COPY src/FileService.Rendering/FileService.Rendering.csproj src/FileService.Rendering/
+COPY src/FileService.Application/FileService.Application.csproj src/FileService.Application/
+COPY src/FileService.Api/FileService.Api.csproj src/FileService.Api/
+RUN dotnet restore src/FileService.Api/FileService.Api.csproj
+COPY src/ src/
+RUN dotnet publish src/FileService.Api/FileService.Api.csproj -c Release -o /app/publish --no-restore
+
+FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS runtime
+WORKDIR /app
+ENV ASPNETCORE_URLS=http://+:8080
+EXPOSE 8080
+COPY --from=build /app/publish .
+ENTRYPOINT ["dotnet", "FileService.Api.dll"]
diff --git a/services/FileService/FileService.slnx b/services/FileService/FileService.slnx
new file mode 100644
index 00000000..eed2251a
--- /dev/null
+++ b/services/FileService/FileService.slnx
@@ -0,0 +1,11 @@
+
+
+
+
+
+
+
+
+
+
+
diff --git a/services/FileService/README.md b/services/FileService/README.md
new file mode 100644
index 00000000..bb6e6a67
--- /dev/null
+++ b/services/FileService/README.md
@@ -0,0 +1,87 @@
+# FileService
+
+Standalone .NET service for file generation and file-related operations: **audit report PDF export** ([QuestPDF](https://www.questpdf.com/)) and **crawl workbook Excel export** (ClosedXML).
+
+Python owns crawl data and CSV/JSON exports. FileService renders PDF and Excel workbooks from the report HTTP API — **no direct Postgres access** (no Npgsql, EF, or `DATABASE_URL`).
+
+## Run locally
+
+Prerequisites: [.NET SDK 10+](https://dotnet.microsoft.com/download), Site Audit report API on port 8001 (started by `./local-run` or Docker `web` service).
+
+```bash
+cd services/FileService
+export REPORT_API_URL=http://127.0.0.1:8001
+dotnet run --project src/FileService.Api
+```
+
+Service listens on **http://localhost:8080** (`ASPNETCORE_URLS` / `appsettings.json`).
+
+In **Development**, Swagger UI is at **http://localhost:8080/docs** and the OpenAPI JSON at **http://localhost:8080/swagger/v1/swagger.json**.
+
+## Environment variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `REPORT_API_URL` | `http://127.0.0.1:8001` | Base URL for report payload, meta, and app-settings HTTP API |
+| `ASPNETCORE_URLS` | `http://127.0.0.1:8080` | Bind address (Docker sets `http://+:8080`) |
+
+## Upstream HTTP contract
+
+Routes on `REPORT_API_URL`:
+
+| Route | Purpose |
+|-------|---------|
+| `GET /api/report/payload?reportId=` | Report JSON for PDF/workbook |
+| `GET /api/report/meta` | Domain → report list |
+| `GET /api/app-settings?key=` | Agency branding (optional) |
+
+## API
+
+| Method | Path | Description |
+|--------|------|-------------|
+| GET | `/health` | Liveness |
+| GET | `/v1/reports/{reportId}/pdf` | PDF for report ID |
+| GET | `/v1/reports/by-domain/{domain}/pdf` | Resolve domain → latest report → PDF |
+| GET | `/v1/reports/{reportId}/workbook` | Excel crawl workbook for report ID |
+| GET | `/v1/reports/by-domain/{domain}/workbook` | Resolve domain → latest report → workbook |
+
+**PDF query params**
+
+| Param | Default | Description |
+|-------|---------|-------------|
+| `profile` | `standard` | `executive`, `standard`, `full`, or `premium` |
+| `branding` | `true` | Load agency name/logo/subtitle from report API app-settings |
+| `disposition` | `attachment` | `inline` for iframe preview, `attachment` for download |
+
+**Workbook query params**
+
+| Param | Default | Description |
+|-------|---------|-------------|
+| `disposition` | `attachment` | `inline` or `attachment` |
+
+Workbook sheets (when data exists in payload): Internal URLs, Links, Redirects, Issues, Custom Fields.
+
+## PDF profiles
+
+| Profile | Contents |
+|---------|----------|
+| **executive** | Branded cover, score dashboard, executive summary, top issues |
+| **standard** | Cover, TOC, score dashboard, audit snapshot, findings, appendix |
+| **full** | Standard sections + Lighthouse, GSC, GA4, security, content, indexation (when data exists) |
+| **premium** | Full-bleed section dividers, enhanced findings with GSC columns, same optional analytics chapters as full |
+
+Layout code lives under `src/FileService.Rendering/Sections/` and `Composition/SectionRegistry.cs`.
+
+## QuestPDF license
+
+QuestPDF Community License applies for companies with annual revenue under USD 1M. Review [QuestPDF licensing](https://www.questpdf.com/license/) before production deployment.
+
+Use the [QuestPDF Companion](https://www.questpdf.com/companion/) app for live layout iteration during development.
+
+## Tests
+
+```bash
+dotnet test
+```
+
+Fixtures: `tests/FileService.Tests/fixtures/minimal-payload.json`, `full-payload.json`.
diff --git a/services/FileService/src/FileService.Api/BinaryFileResult.cs b/services/FileService/src/FileService.Api/BinaryFileResult.cs
new file mode 100644
index 00000000..cb0807f3
--- /dev/null
+++ b/services/FileService/src/FileService.Api/BinaryFileResult.cs
@@ -0,0 +1,13 @@
+namespace FileService.Api;
+
+public sealed class BinaryFileResult(byte[] bytes, string contentType, string contentDisposition) : IResult
+{
+ public Task ExecuteAsync(HttpContext httpContext)
+ {
+ var response = httpContext.Response;
+ response.ContentType = contentType;
+ response.Headers.ContentDisposition = contentDisposition;
+ response.ContentLength = bytes.Length;
+ return response.Body.WriteAsync(bytes).AsTask();
+ }
+}
diff --git a/services/FileService/src/FileService.Api/FileService.Api.csproj b/services/FileService/src/FileService.Api/FileService.Api.csproj
new file mode 100644
index 00000000..658a8f15
--- /dev/null
+++ b/services/FileService/src/FileService.Api/FileService.Api.csproj
@@ -0,0 +1,18 @@
+
+
+
+
+
+
+
+
+
+
+
+
+ net10.0
+ enable
+ enable
+
+
+
diff --git a/services/FileService/src/FileService.Api/OpenApiExtensions.cs b/services/FileService/src/FileService.Api/OpenApiExtensions.cs
new file mode 100644
index 00000000..512223f5
--- /dev/null
+++ b/services/FileService/src/FileService.Api/OpenApiExtensions.cs
@@ -0,0 +1,32 @@
+namespace FileService.Api;
+
+internal static class OpenApiExtensions
+{
+ internal static RouteHandlerBuilder WithPdfOpenApi(
+ this RouteHandlerBuilder builder,
+ string summary,
+ string description)
+ {
+ return builder
+ .WithSummary(summary)
+ .WithDescription(description)
+ .WithTags("Reports")
+ .Produces(StatusCodes.Status200OK, contentType: "application/pdf")
+ .Produces(StatusCodes.Status404NotFound)
+ .Produces(StatusCodes.Status502BadGateway);
+ }
+
+ internal static RouteHandlerBuilder WithWorkbookOpenApi(
+ this RouteHandlerBuilder builder,
+ string summary,
+ string description)
+ {
+ return builder
+ .WithSummary(summary)
+ .WithDescription(description)
+ .WithTags("Reports")
+ .Produces(StatusCodes.Status200OK, contentType: "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
+ .Produces(StatusCodes.Status404NotFound)
+ .Produces(StatusCodes.Status502BadGateway);
+ }
+}
diff --git a/services/FileService/src/FileService.Api/PdfFileResult.cs b/services/FileService/src/FileService.Api/PdfFileResult.cs
new file mode 100644
index 00000000..543be3c9
--- /dev/null
+++ b/services/FileService/src/FileService.Api/PdfFileResult.cs
@@ -0,0 +1,13 @@
+namespace FileService.Api;
+
+public sealed class PdfFileResult(byte[] bytes, string contentDisposition) : IResult
+{
+ public Task ExecuteAsync(HttpContext httpContext)
+ {
+ var response = httpContext.Response;
+ response.ContentType = "application/pdf";
+ response.Headers.ContentDisposition = contentDisposition;
+ response.ContentLength = bytes.Length;
+ return response.Body.WriteAsync(bytes).AsTask();
+ }
+}
diff --git a/services/FileService/src/FileService.Api/Program.cs b/services/FileService/src/FileService.Api/Program.cs
new file mode 100644
index 00000000..b6de222c
--- /dev/null
+++ b/services/FileService/src/FileService.Api/Program.cs
@@ -0,0 +1,187 @@
+using FileService.Api;
+using FileService.Application;
+using FileService.Application.Services;
+using FileService.Domain.Models;
+using Microsoft.OpenApi;
+
+var builder = WebApplication.CreateBuilder(args);
+
+builder.Services.AddFileServiceApplication();
+builder.Services.AddEndpointsApiExplorer();
+builder.Services.AddSwaggerGen(options =>
+{
+ options.SwaggerDoc("v1", new OpenApiInfo
+ {
+ Title = "FileService API",
+ Version = "v1",
+ Description =
+ "PDF export and file generation for Site Audit reports. "
+ + "Fetches report data from the Site Audit report API over HTTP; no direct database access.",
+ });
+});
+
+var app = builder.Build();
+
+if (app.Environment.IsDevelopment())
+{
+ app.UseSwagger();
+ app.UseSwaggerUI(options =>
+ {
+ options.SwaggerEndpoint("/swagger/v1/swagger.json", "FileService API v1");
+ options.RoutePrefix = "docs";
+ });
+}
+
+app.MapGet("/health", () => Results.Ok(new { status = "ok" }))
+ .WithName("HealthCheck")
+ .WithTags("Health")
+ .WithSummary("Liveness probe")
+ .WithDescription("Returns ok when the service is running.");
+
+app.MapGet("/v1/reports/{reportId:int}/pdf", async (
+ int reportId,
+ IPdfReportService pdfService,
+ string? profile,
+ string? disposition,
+ bool? branding,
+ CancellationToken cancellationToken) =>
+{
+ var pdfProfile = ParseProfile(profile);
+ var useBranding = branding ?? true;
+ try
+ {
+ var bytes = await pdfService.GenerateByReportIdAsync(reportId, pdfProfile, useBranding, cancellationToken);
+ return PdfResult(bytes, disposition, $"audit-report-{reportId}.pdf");
+ }
+ catch (KeyNotFoundException ex)
+ {
+ return Results.NotFound(new { detail = ex.Message });
+ }
+ catch (HttpRequestException ex)
+ {
+ return Results.Json(new { detail = "Upstream data service unavailable", error = ex.Message }, statusCode: 502);
+ }
+})
+.WithName("GetReportPdfById")
+.WithPdfOpenApi(
+ "Export report PDF by ID",
+ "Generates a QuestPDF audit report for the given report ID. "
+ + "Query params: profile (executive|standard|full|premium, default standard), "
+ + "branding (default true), disposition (inline|attachment, default attachment).");
+
+app.MapGet("/v1/reports/by-domain/{domain}/pdf", async (
+ string domain,
+ IPdfReportService pdfService,
+ string? profile,
+ string? disposition,
+ bool? branding,
+ CancellationToken cancellationToken) =>
+{
+ var pdfProfile = ParseProfile(profile);
+ var useBranding = branding ?? true;
+ try
+ {
+ var bytes = await pdfService.GenerateByDomainAsync(domain, pdfProfile, useBranding, cancellationToken);
+ var safeName = string.IsNullOrWhiteSpace(domain) ? "report" : domain.Replace('.', '-');
+ return PdfResult(bytes, disposition, $"audit-report-{safeName}.pdf");
+ }
+ catch (KeyNotFoundException ex)
+ {
+ return Results.NotFound(new { detail = ex.Message });
+ }
+ catch (HttpRequestException ex)
+ {
+ return Results.Json(new { detail = "Upstream data service unavailable", error = ex.Message }, statusCode: 502);
+ }
+})
+.WithName("GetReportPdfByDomain")
+.WithPdfOpenApi(
+ "Export report PDF by domain",
+ "Resolves the latest report for the domain, then generates a QuestPDF audit report. "
+ + "Query params: profile (executive|standard|full|premium, default standard), "
+ + "branding (default true), disposition (inline|attachment, default attachment).");
+
+app.MapGet("/v1/reports/{reportId:int}/workbook", async (
+ int reportId,
+ IWorkbookReportService workbookService,
+ string? disposition,
+ CancellationToken cancellationToken) =>
+{
+ try
+ {
+ var bytes = await workbookService.GenerateByReportIdAsync(reportId, cancellationToken);
+ return WorkbookResult(bytes, disposition, $"audit-workbook-{reportId}.xlsx");
+ }
+ catch (KeyNotFoundException ex)
+ {
+ return Results.NotFound(new { detail = ex.Message });
+ }
+ catch (HttpRequestException ex)
+ {
+ return Results.Json(new { detail = "Upstream data service unavailable", error = ex.Message }, statusCode: 502);
+ }
+})
+.WithName("GetReportWorkbookById")
+.WithWorkbookOpenApi(
+ "Export crawl workbook by report ID",
+ "Generates an Excel workbook (Internal URLs, Links, Redirects, Issues, Custom Fields) "
+ + "from the report payload fetched via the report API. disposition: inline|attachment (default attachment).");
+
+app.MapGet("/v1/reports/by-domain/{domain}/workbook", async (
+ string domain,
+ IWorkbookReportService workbookService,
+ string? disposition,
+ CancellationToken cancellationToken) =>
+{
+ try
+ {
+ var bytes = await workbookService.GenerateByDomainAsync(domain, cancellationToken);
+ var safeName = string.IsNullOrWhiteSpace(domain) ? "report" : domain.Replace('.', '-');
+ return WorkbookResult(bytes, disposition, $"audit-workbook-{safeName}.xlsx");
+ }
+ catch (KeyNotFoundException ex)
+ {
+ return Results.NotFound(new { detail = ex.Message });
+ }
+ catch (HttpRequestException ex)
+ {
+ return Results.Json(new { detail = "Upstream data service unavailable", error = ex.Message }, statusCode: 502);
+ }
+})
+.WithName("GetReportWorkbookByDomain")
+.WithWorkbookOpenApi(
+ "Export crawl workbook by domain",
+ "Resolves the latest report for the domain, then generates an Excel crawl workbook.");
+
+app.Run();
+
+static PdfProfile ParseProfile(string? profile) => (profile ?? "standard").Trim().ToLowerInvariant() switch
+{
+ "executive" => PdfProfile.Executive,
+ "full" => PdfProfile.Full,
+ "premium" => PdfProfile.Premium,
+ _ => PdfProfile.Standard,
+};
+
+static IResult PdfResult(byte[] bytes, string? disposition, string filename)
+{
+ var inline = string.Equals(disposition, "inline", StringComparison.OrdinalIgnoreCase);
+ var contentDisposition = inline
+ ? "inline"
+ : $"attachment; filename=\"{filename}\"";
+ return new PdfFileResult(bytes, contentDisposition);
+}
+
+static IResult WorkbookResult(byte[] bytes, string? disposition, string filename)
+{
+ var inline = string.Equals(disposition, "inline", StringComparison.OrdinalIgnoreCase);
+ var contentDisposition = inline
+ ? "inline"
+ : $"attachment; filename=\"{filename}\"";
+ return new BinaryFileResult(
+ bytes,
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+ contentDisposition);
+}
+
+public partial class Program;
diff --git a/services/FileService/src/FileService.Api/Properties/launchSettings.json b/services/FileService/src/FileService.Api/Properties/launchSettings.json
new file mode 100644
index 00000000..f90e2975
--- /dev/null
+++ b/services/FileService/src/FileService.Api/Properties/launchSettings.json
@@ -0,0 +1,24 @@
+{
+ "$schema": "https://json.schemastore.org/launchsettings.json",
+ "profiles": {
+ "http": {
+ "commandName": "Project",
+ "dotnetRunMessages": true,
+ "launchBrowser": true,
+ "launchUrl": "docs",
+ "applicationUrl": "http://localhost:8080",
+ "environmentVariables": {
+ "ASPNETCORE_ENVIRONMENT": "Development"
+ }
+ },
+ "https": {
+ "commandName": "Project",
+ "dotnetRunMessages": true,
+ "launchBrowser": true,
+ "applicationUrl": "https://localhost:7077;http://localhost:5176",
+ "environmentVariables": {
+ "ASPNETCORE_ENVIRONMENT": "Development"
+ }
+ }
+ }
+}
diff --git a/services/FileService/src/FileService.Api/appsettings.Development.json b/services/FileService/src/FileService.Api/appsettings.Development.json
new file mode 100644
index 00000000..ff66ba6b
--- /dev/null
+++ b/services/FileService/src/FileService.Api/appsettings.Development.json
@@ -0,0 +1,8 @@
+{
+ "Logging": {
+ "LogLevel": {
+ "Default": "Information",
+ "Microsoft.AspNetCore": "Warning"
+ }
+ }
+}
diff --git a/services/FileService/src/FileService.Api/appsettings.json b/services/FileService/src/FileService.Api/appsettings.json
new file mode 100644
index 00000000..515b0530
--- /dev/null
+++ b/services/FileService/src/FileService.Api/appsettings.json
@@ -0,0 +1,14 @@
+{
+ "Urls": "http://127.0.0.1:8080",
+ "ReportApi": {
+ "BaseUrl": "http://127.0.0.1:8001",
+ "TimeoutSeconds": 120
+ },
+ "Logging": {
+ "LogLevel": {
+ "Default": "Information",
+ "Microsoft.AspNetCore": "Warning"
+ }
+ },
+ "AllowedHosts": "*"
+}
diff --git a/services/FileService/src/FileService.Application/Clients/AppSettingsClient.cs b/services/FileService/src/FileService.Application/Clients/AppSettingsClient.cs
new file mode 100644
index 00000000..8a17e828
--- /dev/null
+++ b/services/FileService/src/FileService.Application/Clients/AppSettingsClient.cs
@@ -0,0 +1,125 @@
+using System.Net.Http.Json;
+using FileService.Application.Options;
+using FileService.Domain.Models;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace FileService.Application.Clients;
+
+public sealed class AppSettingsClient : IAppSettingsClient
+{
+ private static readonly string[] BrandKeys = ["brand_name", "brand_subtitle", "brand_logo_url"];
+
+ private readonly HttpClient _http;
+ private readonly ILogoFetcher _logoFetcher;
+ private readonly ILogger _logger;
+
+ public AppSettingsClient(
+ HttpClient http,
+ IOptions options,
+ ILogoFetcher logoFetcher,
+ ILogger logger)
+ {
+ _http = http;
+ _logoFetcher = logoFetcher;
+ _logger = logger;
+ _http.BaseAddress = new Uri(options.Value.BaseUrl.TrimEnd('/') + "/");
+ }
+
+ public async Task GetBrandingAsync(bool enabled, CancellationToken cancellationToken = default)
+ {
+ if (!enabled)
+ {
+ return new PdfBrandingModel { Enabled = false };
+ }
+
+ string? name = null;
+ string? subtitle = null;
+ string? logoUrl = null;
+
+ foreach (var key in BrandKeys)
+ {
+ try
+ {
+ using var response = await _http.GetAsync($"api/app-settings?key={key}", cancellationToken);
+ if (!response.IsSuccessStatusCode)
+ {
+ continue;
+ }
+ var data = await response.Content.ReadFromJsonAsync(cancellationToken);
+ var value = data?.Value?.Trim();
+ if (string.IsNullOrEmpty(value))
+ {
+ continue;
+ }
+ switch (key)
+ {
+ case "brand_name": name = value; break;
+ case "brand_subtitle": subtitle = value; break;
+ case "brand_logo_url": logoUrl = value; break;
+ }
+ }
+ catch (Exception ex)
+ {
+ _logger.LogWarning(ex, "Failed to load app setting {Key}", key);
+ }
+ }
+
+ byte[]? logoBytes = null;
+ if (!string.IsNullOrWhiteSpace(logoUrl))
+ {
+ logoBytes = await _logoFetcher.FetchAsync(logoUrl, cancellationToken);
+ }
+
+ return new PdfBrandingModel
+ {
+ Enabled = true,
+ AgencyName = name ?? "",
+ AgencySubtitle = subtitle ?? "",
+ LogoBytes = logoBytes,
+ };
+ }
+
+ private sealed class SettingResponse
+ {
+ public string? Value { get; set; }
+ }
+}
+
+public sealed class LogoFetcher : ILogoFetcher
+{
+ private const int MaxBytes = 512 * 1024;
+ private readonly HttpClient _http;
+ private readonly ILogger _logger;
+
+ public LogoFetcher(HttpClient http, ILogger logger)
+ {
+ _http = http;
+ _logger = logger;
+ }
+
+ public async Task FetchAsync(string? url, CancellationToken cancellationToken = default)
+ {
+ if (string.IsNullOrWhiteSpace(url))
+ {
+ return null;
+ }
+ try
+ {
+ using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+ cts.CancelAfter(TimeSpan.FromSeconds(8));
+ using var response = await _http.GetAsync(url, cts.Token);
+ if (!response.IsSuccessStatusCode)
+ {
+ return null;
+ }
+ var bytes = await response.Content.ReadAsByteArrayAsync(cts.Token);
+ return bytes.Length > MaxBytes ? null : bytes;
+ }
+ catch (Exception ex)
+ {
+ _logger.LogWarning(ex, "Logo fetch failed for {Url}", url);
+ return null;
+ }
+ }
+}
diff --git a/services/FileService/src/FileService.Application/Clients/IAppSettingsClient.cs b/services/FileService/src/FileService.Application/Clients/IAppSettingsClient.cs
new file mode 100644
index 00000000..8b4acbca
--- /dev/null
+++ b/services/FileService/src/FileService.Application/Clients/IAppSettingsClient.cs
@@ -0,0 +1,13 @@
+using FileService.Domain.Models;
+
+namespace FileService.Application.Clients;
+
+public interface IAppSettingsClient
+{
+ Task GetBrandingAsync(bool enabled, CancellationToken cancellationToken = default);
+}
+
+public interface ILogoFetcher
+{
+ Task FetchAsync(string? url, CancellationToken cancellationToken = default);
+}
diff --git a/services/FileService/src/FileService.Application/Clients/IReportDataClient.cs b/services/FileService/src/FileService.Application/Clients/IReportDataClient.cs
new file mode 100644
index 00000000..5d74c905
--- /dev/null
+++ b/services/FileService/src/FileService.Application/Clients/IReportDataClient.cs
@@ -0,0 +1,10 @@
+using System.Text.Json;
+using FileService.Domain.Models;
+
+namespace FileService.Application.Clients;
+
+public interface IReportDataClient
+{
+ Task> ListReportsAsync(CancellationToken cancellationToken = default);
+ Task GetPayloadAsync(int reportId, CancellationToken cancellationToken = default);
+}
diff --git a/services/FileService/src/FileService.Application/Clients/ReportDataClient.cs b/services/FileService/src/FileService.Application/Clients/ReportDataClient.cs
new file mode 100644
index 00000000..5780b9f3
--- /dev/null
+++ b/services/FileService/src/FileService.Application/Clients/ReportDataClient.cs
@@ -0,0 +1,90 @@
+using System.Net;
+using System.Text.Json;
+using FileService.Application.Options;
+using FileService.Domain.Models;
+using Microsoft.Extensions.Logging;
+using Microsoft.Extensions.Options;
+
+namespace FileService.Application.Clients;
+
+public sealed class ReportDataClient : IReportDataClient
+{
+ private static readonly JsonSerializerOptions JsonOptions = new()
+ {
+ PropertyNameCaseInsensitive = true,
+ };
+
+ private readonly HttpClient _http;
+ private readonly ILogger _logger;
+
+ public ReportDataClient(HttpClient http, IOptions options, ILogger logger)
+ {
+ _http = http;
+ _logger = logger;
+ var baseUrl = options.Value.BaseUrl.TrimEnd('/');
+ _http.BaseAddress = new Uri(baseUrl + "/");
+ _http.Timeout = TimeSpan.FromSeconds(Math.Max(5, options.Value.TimeoutSeconds));
+ }
+
+ public async Task> ListReportsAsync(CancellationToken cancellationToken = default)
+ {
+ using var response = await _http.GetAsync("api/report/meta", cancellationToken);
+ await EnsureSuccessAsync(response, cancellationToken);
+ await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken);
+ using var doc = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken);
+ if (!doc.RootElement.TryGetProperty("reports", out var reportsEl) || reportsEl.ValueKind != JsonValueKind.Array)
+ {
+ return [];
+ }
+
+ var rows = new List();
+ foreach (var item in reportsEl.EnumerateArray())
+ {
+ rows.Add(new ReportListRow
+ {
+ Id = item.TryGetProperty("id", out var idEl) ? idEl.GetInt32() : 0,
+ CanonicalDomain = GetString(item, "canonical_domain"),
+ SiteName = GetString(item, "site_name"),
+ GeneratedAt = GetString(item, "generated_at"),
+ });
+ }
+ return rows;
+ }
+
+ public async Task GetPayloadAsync(int reportId, CancellationToken cancellationToken = default)
+ {
+ using var response = await _http.GetAsync($"api/report/payload?reportId={reportId}", cancellationToken);
+ if (response.StatusCode == HttpStatusCode.NotFound)
+ {
+ return null;
+ }
+ await EnsureSuccessAsync(response, cancellationToken);
+ await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken);
+ using var doc = await JsonDocument.ParseAsync(stream, cancellationToken: cancellationToken);
+ if (!doc.RootElement.TryGetProperty("payload", out var payload))
+ {
+ return null;
+ }
+ return payload.Clone();
+ }
+
+ private static string? GetString(JsonElement el, string name)
+ {
+ if (!el.TryGetProperty(name, out var prop) || prop.ValueKind == JsonValueKind.Null)
+ {
+ return null;
+ }
+ return prop.GetString();
+ }
+
+ private async Task EnsureSuccessAsync(HttpResponseMessage response, CancellationToken cancellationToken)
+ {
+ if (response.IsSuccessStatusCode)
+ {
+ return;
+ }
+ var body = await response.Content.ReadAsStringAsync(cancellationToken);
+ _logger.LogWarning("Report API request failed: {Status} {Body}", response.StatusCode, body);
+ throw new HttpRequestException($"Report API returned {(int)response.StatusCode}: {body}", null, response.StatusCode);
+ }
+}
diff --git a/services/FileService/src/FileService.Application/DependencyInjection.cs b/services/FileService/src/FileService.Application/DependencyInjection.cs
new file mode 100644
index 00000000..ed35ea61
--- /dev/null
+++ b/services/FileService/src/FileService.Application/DependencyInjection.cs
@@ -0,0 +1,33 @@
+using FileService.Application.Clients;
+using FileService.Application.Options;
+using FileService.Application.Services;
+using FileService.Rendering;
+using Microsoft.Extensions.DependencyInjection;
+
+namespace FileService.Application;
+
+public static class DependencyInjection
+{
+ public static IServiceCollection AddFileServiceApplication(this IServiceCollection services)
+ {
+ services.AddOptions()
+ .BindConfiguration(ReportApiOptions.SectionName)
+ .PostConfigure(o =>
+ {
+ var env = Environment.GetEnvironmentVariable("REPORT_API_URL");
+ if (!string.IsNullOrWhiteSpace(env))
+ {
+ o.BaseUrl = env.Trim();
+ }
+ });
+
+ services.AddHttpClient();
+ services.AddHttpClient();
+ services.AddHttpClient();
+ services.AddSingleton();
+ services.AddSingleton();
+ services.AddScoped();
+ services.AddScoped();
+ return services;
+ }
+}
diff --git a/services/FileService/src/FileService.Application/Domain/DomainResolver.cs b/services/FileService/src/FileService.Application/Domain/DomainResolver.cs
new file mode 100644
index 00000000..163a131a
--- /dev/null
+++ b/services/FileService/src/FileService.Application/Domain/DomainResolver.cs
@@ -0,0 +1,84 @@
+using System.Text.RegularExpressions;
+using FileService.Domain.Models;
+
+namespace FileService.Application.Domain;
+
+public static partial class DomainResolver
+{
+ public static int? ResolveReportId(IReadOnlyList reports, string domainQuery)
+ {
+ var query = NormalizeDomainQuery(domainQuery);
+ if (string.IsNullOrEmpty(query))
+ {
+ return null;
+ }
+
+ foreach (var row in reports)
+ {
+ if (DomainQueryMatchesRow(row, query))
+ {
+ return row.Id;
+ }
+ }
+ return null;
+ }
+
+ public static bool DomainQueryMatchesRow(ReportListRow row, string queryParam)
+ {
+ var p = NormalizeDomainQuery(queryParam);
+ if (string.IsNullOrEmpty(p))
+ {
+ return false;
+ }
+
+ var host = row.CanonicalDomain?.Trim().ToLowerInvariant() ?? "";
+ if (!string.IsNullOrEmpty(host) && host == p)
+ {
+ return true;
+ }
+ if (Slugify(row.SiteName) == p)
+ {
+ return true;
+ }
+ if (!string.IsNullOrEmpty(host) && Slugify(host) == p)
+ {
+ return true;
+ }
+ if (!string.IsNullOrEmpty(row.SiteName) && row.SiteName.Trim().ToLowerInvariant() == p)
+ {
+ return true;
+ }
+ return false;
+ }
+
+ public static string NormalizeDomainQuery(string? param)
+ {
+ if (string.IsNullOrWhiteSpace(param))
+ {
+ return "";
+ }
+ var s = Uri.UnescapeDataString(param.Trim()).ToLowerInvariant();
+ return TrailingPunctuation().Replace(s, "");
+ }
+
+ public static string Slugify(string? name)
+ {
+ if (string.IsNullOrWhiteSpace(name))
+ {
+ return "";
+ }
+ var s = name.Trim().ToLowerInvariant();
+ s = NonAlphaNum().Replace(s, "-");
+ s = TrimHyphens().Replace(s, "");
+ return s;
+ }
+
+ [GeneratedRegex(@"[,;.\s]+$")]
+ private static partial Regex TrailingPunctuation();
+
+ [GeneratedRegex(@"[^a-z0-9]+")]
+ private static partial Regex NonAlphaNum();
+
+ [GeneratedRegex(@"^-+|-+$")]
+ private static partial Regex TrimHyphens();
+}
diff --git a/services/FileService/src/FileService.Application/FileService.Application.csproj b/services/FileService/src/FileService.Application/FileService.Application.csproj
new file mode 100644
index 00000000..9a212f4c
--- /dev/null
+++ b/services/FileService/src/FileService.Application/FileService.Application.csproj
@@ -0,0 +1,19 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+ net10.0
+ enable
+ enable
+
+
+
diff --git a/services/FileService/src/FileService.Application/Mapping/AuditReportMapper.cs b/services/FileService/src/FileService.Application/Mapping/AuditReportMapper.cs
new file mode 100644
index 00000000..012e6edd
--- /dev/null
+++ b/services/FileService/src/FileService.Application/Mapping/AuditReportMapper.cs
@@ -0,0 +1,352 @@
+using System.Globalization;
+using System.Text.Json;
+using FileService.Domain.Models;
+
+namespace FileService.Application.Mapping;
+
+public static class AuditReportMapper
+{
+ private const int IssuesTotalLimit = 120;
+ private const int IssuesPerGroupLimit = 25;
+ private const int TopIssuesCoverLimit = 6;
+
+ private static readonly Dictionary PriorityOrder = new(StringComparer.OrdinalIgnoreCase)
+ {
+ ["critical"] = 0,
+ ["high"] = 1,
+ ["medium"] = 2,
+ ["low"] = 3,
+ };
+
+ public static AuditReportModel Map(
+ JsonElement payload,
+ int reportId,
+ PdfProfile profile,
+ PdfBrandingModel branding)
+ {
+ var exportedAt = DateTime.UtcNow.ToString("dd MMMM yyyy, HH:mm 'UTC'", CultureInfo.InvariantCulture);
+ var allIssues = ExtractIssues(payload);
+ var issueCounts = CountByPriority(allIssues);
+ var healthScore = ComputeOverallScore(payload);
+ var exec = ExtractExecutiveSummary(payload, allIssues);
+ var limitedIssues = LimitIssues(allIssues, profile);
+ var truncationNotes = BuildTruncationNotes(allIssues, limitedIssues);
+ var includeChapters = profile is PdfProfile.Full or PdfProfile.Premium;
+
+ return new AuditReportModel
+ {
+ ReportId = reportId,
+ SiteName = JsonHelper.GetString(payload, "site_name") ?? "Site",
+ ReportTitle = JsonHelper.GetString(payload, "report_title") ?? "Technical SEO Audit Report",
+ GeneratedAt = FormatReportDate(JsonHelper.GetString(payload, "report_generated_at")),
+ ExportedAt = exportedAt,
+ HealthScore = healthScore,
+ ScoreBand = ScoreBand(healthScore),
+ TotalIssueCount = allIssues.Count,
+ DataSources = ExtractDataSources(payload),
+ Branding = branding,
+ ExecutiveSummary = exec,
+ CategoryScores = ExtractCategoryScores(payload),
+ Issues = limitedIssues,
+ IssueCounts = issueCounts,
+ Snapshot = ChapterMappers.MapSnapshot(payload),
+ Lighthouse = includeChapters ? ChapterMappers.MapLighthouse(payload) : null,
+ SearchVisibility = includeChapters ? ChapterMappers.MapSearchVisibility(payload) : null,
+ Traffic = includeChapters ? ChapterMappers.MapTraffic(payload) : null,
+ Security = includeChapters ? ChapterMappers.MapSecurity(payload) : null,
+ Content = includeChapters ? ChapterMappers.MapContent(payload) : null,
+ Indexation = includeChapters ? ChapterMappers.MapIndexation(payload) : null,
+ LinkSamples = ChapterMappers.MapLinkSamples(payload),
+ TruncationNotes = truncationNotes,
+ CrawlScope = ExtractCrawlScope(payload),
+ };
+ }
+
+ private static List ExtractIssues(JsonElement payload)
+ {
+ var rows = new List();
+ if (!payload.TryGetProperty("categories", out var categories) || categories.ValueKind != JsonValueKind.Array)
+ {
+ return rows;
+ }
+
+ foreach (var cat in categories.EnumerateArray())
+ {
+ var catName = JsonHelper.GetString(cat, "name") ?? "";
+ if (!cat.TryGetProperty("issues", out var issues) || issues.ValueKind != JsonValueKind.Array)
+ {
+ continue;
+ }
+ foreach (var issue in issues.EnumerateArray())
+ {
+ var rule = JsonHelper.GetString(issue, "recommendation") ?? "";
+ var llm = JsonHelper.GetString(issue, "llm_recommendation") ?? "";
+ var rec = !string.IsNullOrWhiteSpace(llm) ? llm : rule;
+ rows.Add(IssueNormalizer.Normalize(
+ CategoryDisplayName(catName),
+ JsonHelper.GetString(issue, "priority") ?? "",
+ JsonHelper.GetString(issue, "message") ?? "",
+ JsonHelper.GetString(issue, "url") ?? "",
+ rec,
+ JsonHelper.GetInt(issue, "gsc_clicks"),
+ JsonHelper.GetInt(issue, "gsc_impressions"),
+ JsonHelper.GetInt(issue, "impact_score")));
+ }
+ }
+
+ rows.Sort((a, b) =>
+ {
+ var pa = PriorityOrder.GetValueOrDefault(a.Priority.ToLowerInvariant(), 9);
+ var pb = PriorityOrder.GetValueOrDefault(b.Priority.ToLowerInvariant(), 9);
+ return pa != pb ? pa.CompareTo(pb) : string.Compare(a.Headline, b.Headline, StringComparison.Ordinal);
+ });
+ return rows;
+ }
+
+ private static IReadOnlyList LimitIssues(List allIssues, PdfProfile profile)
+ {
+ var max = profile switch
+ {
+ PdfProfile.Executive => TopIssuesCoverLimit,
+ _ => IssuesTotalLimit,
+ };
+
+ if (profile == PdfProfile.Executive)
+ {
+ return allIssues.Take(max).ToList();
+ }
+
+ var result = new List();
+ var perGroup = new Dictionary(StringComparer.OrdinalIgnoreCase);
+ foreach (var issue in allIssues)
+ {
+ if (result.Count >= max)
+ {
+ break;
+ }
+ var key = issue.Priority.ToLowerInvariant();
+ perGroup.TryGetValue(key, out var count);
+ if (count >= IssuesPerGroupLimit)
+ {
+ continue;
+ }
+ perGroup[key] = count + 1;
+ result.Add(issue);
+ }
+ return result;
+ }
+
+ private static ExecutiveSummaryModel ExtractExecutiveSummary(JsonElement payload, List allIssues)
+ {
+ var summary = "";
+ var source = "";
+ var priorities = new List();
+ var topIssues = new List();
+
+ if (payload.TryGetProperty("executive_summary", out var exec) && exec.ValueKind == JsonValueKind.Object)
+ {
+ summary = JsonHelper.GetString(exec, "summary") ?? "";
+ source = ExecutiveSourceLabel(JsonHelper.GetString(exec, "source"));
+ if (exec.TryGetProperty("priorities", out var priEl) && priEl.ValueKind == JsonValueKind.Array)
+ {
+ foreach (var p in priEl.EnumerateArray())
+ {
+ var s = p.GetString()?.Trim();
+ if (!string.IsNullOrEmpty(s))
+ {
+ priorities.Add(s);
+ }
+ }
+ }
+ if (exec.TryGetProperty("top_issues", out var topEl) && topEl.ValueKind == JsonValueKind.Array)
+ {
+ foreach (var issue in topEl.EnumerateArray().Take(8))
+ {
+ topIssues.Add(IssueNormalizer.Normalize(
+ JsonHelper.GetString(issue, "category") ?? "",
+ JsonHelper.GetString(issue, "priority") ?? "",
+ JsonHelper.GetString(issue, "message") ?? "",
+ JsonHelper.GetString(issue, "url") ?? "",
+ JsonHelper.GetString(issue, "recommendation") ?? "",
+ null, null, null));
+ }
+ }
+ }
+
+ if (string.IsNullOrWhiteSpace(summary) && payload.TryGetProperty("recommendations", out var recs) && recs.ValueKind == JsonValueKind.Array)
+ {
+ var legacy = recs.EnumerateArray()
+ .Select(r => r.GetString()?.Trim())
+ .Where(s => !string.IsNullOrEmpty(s))
+ .Take(12)
+ .Select(s => $"• {s}")
+ .ToList();
+ if (legacy.Count > 0)
+ {
+ summary = string.Join("\n", legacy);
+ }
+ }
+
+ if (topIssues.Count == 0)
+ {
+ topIssues = allIssues.Take(TopIssuesCoverLimit).ToList();
+ }
+
+ return new ExecutiveSummaryModel
+ {
+ Summary = summary,
+ SourceLabel = source,
+ Priorities = priorities,
+ TopIssues = topIssues,
+ };
+ }
+
+ private static IReadOnlyList ExtractCategoryScores(JsonElement payload)
+ {
+ var scores = new List();
+ if (!payload.TryGetProperty("categories", out var categories) || categories.ValueKind != JsonValueKind.Array)
+ {
+ return scores;
+ }
+ foreach (var cat in categories.EnumerateArray())
+ {
+ int issueCount = 0;
+ if (cat.TryGetProperty("issues", out var issues) && issues.ValueKind == JsonValueKind.Array)
+ {
+ issueCount = issues.GetArrayLength();
+ }
+ scores.Add(new CategoryScoreModel
+ {
+ Name = CategoryDisplayName(JsonHelper.GetString(cat, "name") ?? ""),
+ Score = JsonHelper.GetInt(cat, "score"),
+ IssueCount = issueCount,
+ });
+ }
+ return scores;
+ }
+
+ private static CrawlScopeModel? ExtractCrawlScope(JsonElement payload)
+ {
+ if (!payload.TryGetProperty("report_meta", out var meta) || meta.ValueKind != JsonValueKind.Object)
+ {
+ return null;
+ }
+ if (!meta.TryGetProperty("crawl_scope", out var scope) || scope.ValueKind != JsonValueKind.Object)
+ {
+ return null;
+ }
+ return new CrawlScopeModel
+ {
+ PagesCrawled = JsonHelper.GetInt(scope, "pages_crawled"),
+ MaxPagesConfigured = JsonHelper.GetInt(scope, "max_pages_configured"),
+ };
+ }
+
+ private static IReadOnlyList ExtractDataSources(JsonElement payload)
+ {
+ if (!payload.TryGetProperty("report_meta", out var meta) || meta.ValueKind != JsonValueKind.Object)
+ {
+ return [];
+ }
+ if (!meta.TryGetProperty("data_sources", out var sources) || sources.ValueKind != JsonValueKind.Array)
+ {
+ return [];
+ }
+ return sources.EnumerateArray()
+ .Select(s => s.GetString())
+ .Where(s => !string.IsNullOrWhiteSpace(s))
+ .Select(s => s!)
+ .ToList();
+ }
+
+ private static IReadOnlyList BuildTruncationNotes(List all, IReadOnlyList limited)
+ {
+ if (all.Count <= limited.Count)
+ {
+ return [];
+ }
+ return [$"Showing {limited.Count} of {all.Count} issues — export CSV for the full list."];
+ }
+
+ private static Dictionary CountByPriority(IEnumerable issues)
+ {
+ var counts = new Dictionary(StringComparer.OrdinalIgnoreCase)
+ {
+ ["critical"] = 0,
+ ["high"] = 0,
+ ["medium"] = 0,
+ ["low"] = 0,
+ };
+ foreach (var issue in issues)
+ {
+ var key = issue.Priority.ToLowerInvariant();
+ if (counts.ContainsKey(key))
+ {
+ counts[key]++;
+ }
+ }
+ return counts;
+ }
+
+ private static int? ComputeOverallScore(JsonElement payload)
+ {
+ if (payload.TryGetProperty("overall_score", out var overall) && overall.ValueKind == JsonValueKind.Number)
+ {
+ return (int)Math.Round(overall.GetDouble());
+ }
+ if (!payload.TryGetProperty("categories", out var categories) || categories.ValueKind != JsonValueKind.Array)
+ {
+ return null;
+ }
+ var scores = new List();
+ foreach (var cat in categories.EnumerateArray())
+ {
+ if (cat.TryGetProperty("score", out var scoreEl) && scoreEl.ValueKind == JsonValueKind.Number)
+ {
+ scores.Add(scoreEl.GetDouble());
+ }
+ }
+ return scores.Count == 0 ? null : (int)Math.Round(scores.Average());
+ }
+
+ private static string ScoreBand(int? score) => score switch
+ {
+ >= 80 => "Excellent",
+ >= 60 => "Good",
+ >= 40 => "Needs work",
+ _ => "Critical",
+ };
+
+ private static string ExecutiveSourceLabel(string? source) => source switch
+ {
+ "ai_insights" => "AI insights",
+ "deterministic" => "Measured + Search Console",
+ _ => source ?? "Audit data",
+ };
+
+ private static string CategoryDisplayName(string name) => name switch
+ {
+ "technical_seo" => "Technical SEO",
+ "content" => "Content",
+ "performance" => "Performance",
+ "security" => "Security",
+ "indexation" => "Indexation",
+ "links" => "Links",
+ _ => string.IsNullOrWhiteSpace(name)
+ ? "General"
+ : CultureInfo.InvariantCulture.TextInfo.ToTitleCase(name.Replace('_', ' ')),
+ };
+
+ private static string FormatReportDate(string? raw)
+ {
+ if (string.IsNullOrWhiteSpace(raw))
+ {
+ return "";
+ }
+ if (DateTime.TryParse(raw, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal, out var dt))
+ {
+ return dt.ToUniversalTime().ToString("dd MMMM yyyy", CultureInfo.InvariantCulture);
+ }
+ return raw;
+ }
+}
diff --git a/services/FileService/src/FileService.Application/Mapping/ChapterMappers.cs b/services/FileService/src/FileService.Application/Mapping/ChapterMappers.cs
new file mode 100644
index 00000000..ea7829b1
--- /dev/null
+++ b/services/FileService/src/FileService.Application/Mapping/ChapterMappers.cs
@@ -0,0 +1,247 @@
+using System.Text.Json;
+using FileService.Domain.Models;
+
+namespace FileService.Application.Mapping;
+
+public static class ChapterMappers
+{
+ public static AuditSnapshotModel? MapSnapshot(JsonElement payload)
+ {
+ var summary = payload.TryGetProperty("summary", out var s) && s.ValueKind == JsonValueKind.Object ? s : default;
+ var hasSummary = summary.ValueKind == JsonValueKind.Object;
+ var statusCounts = ExtractStatusCounts(payload);
+ var renderMode = payload.TryGetProperty("report_meta", out var meta) && meta.ValueKind == JsonValueKind.Object
+ && meta.TryGetProperty("crawl_scope", out var scope) && scope.ValueKind == JsonValueKind.Object
+ ? JsonHelper.GetString(scope, "render_mode")
+ : null;
+
+ if (!hasSummary && statusCounts.Count == 0 && renderMode is null)
+ {
+ return null;
+ }
+
+ return new AuditSnapshotModel
+ {
+ TotalUrls = hasSummary ? JsonHelper.GetInt(summary, "total_urls") : null,
+ IndexableUrls = hasSummary ? JsonHelper.GetInt(summary, "indexable") : null,
+ TotalIssues = hasSummary ? JsonHelper.GetInt(summary, "total_issues") : null,
+ CriticalIssues = hasSummary ? JsonHelper.GetInt(summary, "critical_issues") : null,
+ StatusCounts = statusCounts,
+ GoogleFetchedAt = meta.ValueKind == JsonValueKind.Object ? JsonHelper.GetString(meta, "google_fetched_at") : null,
+ RenderMode = renderMode,
+ };
+ }
+
+ public static LighthouseChapterModel? MapLighthouse(JsonElement payload)
+ {
+ if (!payload.TryGetProperty("lighthouse_summary", out var lh) || lh.ValueKind != JsonValueKind.Object)
+ {
+ return null;
+ }
+ var human = JsonHelper.GetString(payload, "lighthouse_human_summary") ?? "";
+ var diagnostics = new List();
+ if (payload.TryGetProperty("lighthouse_diagnostics", out var diagEl) && diagEl.ValueKind == JsonValueKind.Array)
+ {
+ foreach (var d in diagEl.EnumerateArray().Take(15))
+ {
+ diagnostics.Add(new LighthouseDiagnosticModel
+ {
+ Title = JsonHelper.GetString(d, "title") ?? JsonHelper.GetString(d, "id") ?? "",
+ Description = JsonHelper.GetString(d, "description") ?? "",
+ });
+ }
+ }
+ return new LighthouseChapterModel
+ {
+ Summary = new LighthouseSummaryModel
+ {
+ Url = JsonHelper.GetString(lh, "url") ?? "",
+ Performance = JsonHelper.GetInt(lh, "performance"),
+ Accessibility = JsonHelper.GetInt(lh, "accessibility"),
+ BestPractices = JsonHelper.GetInt(lh, "best_practices"),
+ Seo = JsonHelper.GetInt(lh, "seo"),
+ },
+ HumanSummary = human,
+ Diagnostics = diagnostics,
+ };
+ }
+
+ public static SearchVisibilityModel? MapSearchVisibility(JsonElement payload)
+ {
+ if (!payload.TryGetProperty("search_performance", out var sp) || sp.ValueKind != JsonValueKind.Object)
+ {
+ if (!payload.TryGetProperty("gsc", out var gsc) || gsc.ValueKind != JsonValueKind.Object)
+ {
+ return null;
+ }
+ sp = gsc;
+ }
+ var queries = MapMetricRows(sp, "queries", "query", "clicks", "impressions");
+ var pages = MapMetricRows(sp, "pages", "page", "clicks", "impressions");
+ if (queries.Count == 0 && pages.Count == 0)
+ {
+ return null;
+ }
+ return new SearchVisibilityModel { TopQueries = queries, TopPages = pages };
+ }
+
+ public static TrafficSnapshotModel? MapTraffic(JsonElement payload)
+ {
+ if (!payload.TryGetProperty("ga4", out var ga4) || ga4.ValueKind != JsonValueKind.Object)
+ {
+ return null;
+ }
+ var channels = MapMetricRows(ga4, "channels", "channel", "sessions");
+ var devices = MapMetricRows(ga4, "devices", "device", "sessions");
+ if (channels.Count == 0 && devices.Count == 0)
+ {
+ return null;
+ }
+ return new TrafficSnapshotModel { Channels = channels, Devices = devices };
+ }
+
+ public static SecurityChapterModel? MapSecurity(JsonElement payload)
+ {
+ if (!payload.TryGetProperty("security_findings", out var arr) || arr.ValueKind != JsonValueKind.Array)
+ {
+ return null;
+ }
+ var findings = arr.EnumerateArray().Take(25).Select(f => new SecurityFindingModel
+ {
+ Severity = JsonHelper.GetString(f, "severity") ?? "medium",
+ Type = JsonHelper.GetString(f, "finding_type") ?? JsonHelper.GetString(f, "type") ?? "",
+ Url = JsonHelper.GetString(f, "url") ?? "",
+ Message = JsonHelper.GetString(f, "message") ?? "",
+ }).ToList();
+ return findings.Count == 0 ? null : new SecurityChapterModel { Findings = findings };
+ }
+
+ public static ContentChapterModel? MapContent(JsonElement payload)
+ {
+ if (!payload.TryGetProperty("content_analytics", out var ca) || ca.ValueKind != JsonValueKind.Object)
+ {
+ return null;
+ }
+ var stats = ca.TryGetProperty("word_count_stats", out var wcs) && wcs.ValueKind == JsonValueKind.Object ? wcs : default;
+ var keywords = new List();
+ if (ca.TryGetProperty("top_keywords_site", out var kwEl) && kwEl.ValueKind == JsonValueKind.Array)
+ {
+ foreach (var kw in kwEl.EnumerateArray().Take(15))
+ {
+ keywords.Add(new MetricRowModel
+ {
+ Label = JsonHelper.GetString(kw, "word") ?? "",
+ Value = JsonHelper.GetString(kw, "count") ?? "",
+ });
+ }
+ }
+ return new ContentChapterModel
+ {
+ MeanWordCount = stats.ValueKind == JsonValueKind.Object ? JsonHelper.GetInt(stats, "mean") : null,
+ MedianWordCount = stats.ValueKind == JsonValueKind.Object ? JsonHelper.GetInt(stats, "median") : null,
+ ThinContentCount = JsonHelper.GetInt(ca, "thin_content_count"),
+ TopKeywords = keywords,
+ };
+ }
+
+ public static IndexationChapterModel? MapIndexation(JsonElement payload)
+ {
+ if (!payload.TryGetProperty("indexation_coverage", out var ic) || ic.ValueKind != JsonValueKind.Object)
+ {
+ return null;
+ }
+ return new IndexationChapterModel
+ {
+ Indexable = JsonHelper.GetInt(ic, "indexable"),
+ NonIndexable = JsonHelper.GetInt(ic, "non_indexable"),
+ Blocked = JsonHelper.GetInt(ic, "blocked"),
+ Notes = JsonHelper.GetString(ic, "notes"),
+ };
+ }
+
+ public static IReadOnlyList MapLinkSamples(JsonElement payload, int limit = 20)
+ {
+ if (!payload.TryGetProperty("links", out var links) || links.ValueKind != JsonValueKind.Array)
+ {
+ return [];
+ }
+ return links.EnumerateArray().Take(limit).Select(l => new LinkSampleModel
+ {
+ Url = JsonHelper.GetString(l, "url") ?? "",
+ Status = JsonHelper.GetString(l, "status") ?? "",
+ Title = JsonHelper.GetString(l, "title") ?? "",
+ }).Where(l => !string.IsNullOrWhiteSpace(l.Url)).ToList();
+ }
+
+ private static List MapMetricRows(
+ JsonElement parent,
+ string arrayName,
+ string labelKey,
+ string valueKey,
+ string? secondaryKey = null)
+ {
+ var rows = new List();
+ if (!parent.TryGetProperty(arrayName, out var arr) || arr.ValueKind != JsonValueKind.Array)
+ {
+ return rows;
+ }
+ foreach (var item in arr.EnumerateArray().Take(10))
+ {
+ var label = JsonHelper.GetString(item, labelKey) ?? JsonHelper.GetString(item, "url") ?? "";
+ if (string.IsNullOrWhiteSpace(label))
+ {
+ continue;
+ }
+ rows.Add(new MetricRowModel
+ {
+ Label = label,
+ Value = JsonHelper.GetString(item, valueKey) ?? "0",
+ Secondary = secondaryKey is not null ? JsonHelper.GetString(item, secondaryKey) : null,
+ });
+ }
+ return rows;
+ }
+
+ private static Dictionary ExtractStatusCounts(JsonElement payload)
+ {
+ var result = new Dictionary();
+ if (!payload.TryGetProperty("status_counts", out var sc) || sc.ValueKind != JsonValueKind.Object)
+ {
+ return result;
+ }
+ foreach (var prop in sc.EnumerateObject())
+ {
+ if (prop.Value.ValueKind == JsonValueKind.Number)
+ {
+ result[prop.Name] = (int)Math.Round(prop.Value.GetDouble());
+ }
+ }
+ return result;
+ }
+}
+
+internal static class JsonHelper
+{
+ public static string? GetString(JsonElement el, string name)
+ {
+ if (!el.TryGetProperty(name, out var prop))
+ {
+ return null;
+ }
+ return prop.ValueKind switch
+ {
+ JsonValueKind.String => prop.GetString(),
+ JsonValueKind.Number => prop.GetRawText(),
+ _ => null,
+ };
+ }
+
+ public static int? GetInt(JsonElement el, string name)
+ {
+ if (!el.TryGetProperty(name, out var prop) || prop.ValueKind != JsonValueKind.Number)
+ {
+ return null;
+ }
+ return (int)Math.Round(prop.GetDouble());
+ }
+}
diff --git a/services/FileService/src/FileService.Application/Mapping/IssueNormalizer.cs b/services/FileService/src/FileService.Application/Mapping/IssueNormalizer.cs
new file mode 100644
index 00000000..c984476e
--- /dev/null
+++ b/services/FileService/src/FileService.Application/Mapping/IssueNormalizer.cs
@@ -0,0 +1,77 @@
+using System.Text.RegularExpressions;
+using FileService.Domain.Models;
+
+namespace FileService.Application.Mapping;
+
+public static partial class IssueNormalizer
+{
+ public static IssueModel Normalize(
+ string category,
+ string priority,
+ string message,
+ string url,
+ string recommendation,
+ int? gscClicks,
+ int? gscImpressions,
+ int? impactScore)
+ {
+ var headline = NormalizeHeadline(message, url);
+ return new IssueModel
+ {
+ Category = category,
+ Priority = priority,
+ Message = message,
+ Headline = headline,
+ Url = url,
+ UrlPath = ExtractPath(url),
+ Recommendation = recommendation,
+ GscClicks = gscClicks,
+ GscImpressions = gscImpressions,
+ ImpactScore = impactScore,
+ };
+ }
+
+ public static string NormalizeHeadline(string message, string url)
+ {
+ var headline = message.Trim();
+ if (string.IsNullOrWhiteSpace(url))
+ {
+ return Truncate(headline, 160);
+ }
+ try
+ {
+ var path = new Uri(url.StartsWith("http") ? url : $"https://{url}").AbsolutePath;
+ if (headline.Contains(path, StringComparison.OrdinalIgnoreCase))
+ {
+ headline = headline.Replace(path, "", StringComparison.OrdinalIgnoreCase).Trim(' ', '-', ':');
+ }
+ }
+ catch
+ {
+ // keep headline
+ }
+ return Truncate(headline, 160);
+ }
+
+ public static string ExtractPath(string url)
+ {
+ if (string.IsNullOrWhiteSpace(url))
+ {
+ return "";
+ }
+ try
+ {
+ return new Uri(url.StartsWith("http") ? url : $"https://{url}").AbsolutePath;
+ }
+ catch
+ {
+ return url;
+ }
+ }
+
+ private static string Truncate(string s, int max) =>
+ s.Length <= max ? s : s[..(max - 1)] + "…";
+
+ [GeneratedRegex(@"\s+")]
+ private static partial Regex Whitespace();
+}
diff --git a/services/FileService/src/FileService.Application/Options/ReportApiOptions.cs b/services/FileService/src/FileService.Application/Options/ReportApiOptions.cs
new file mode 100644
index 00000000..c32dced4
--- /dev/null
+++ b/services/FileService/src/FileService.Application/Options/ReportApiOptions.cs
@@ -0,0 +1,13 @@
+namespace FileService.Application.Options;
+
+///
+/// HTTP base URL for the Site Audit report API (JSON payload, meta, app-settings).
+/// FileService is not tied to any specific backend framework — only this HTTP contract.
+///
+public sealed class ReportApiOptions
+{
+ public const string SectionName = "ReportApi";
+
+ public string BaseUrl { get; set; } = "http://127.0.0.1:8001";
+ public int TimeoutSeconds { get; set; } = 120;
+}
diff --git a/services/FileService/src/FileService.Application/Services/PdfReportService.cs b/services/FileService/src/FileService.Application/Services/PdfReportService.cs
new file mode 100644
index 00000000..e47ddbab
--- /dev/null
+++ b/services/FileService/src/FileService.Application/Services/PdfReportService.cs
@@ -0,0 +1,75 @@
+using FileService.Application.Clients;
+using FileService.Application.Domain;
+using FileService.Application.Mapping;
+using FileService.Domain.Models;
+using FileService.Rendering;
+using FileService.Rendering.Composition;
+using Microsoft.Extensions.Logging;
+
+namespace FileService.Application.Services;
+
+public interface IPdfReportService
+{
+ Task GenerateByReportIdAsync(
+ int reportId,
+ PdfProfile profile,
+ bool branding,
+ CancellationToken cancellationToken = default);
+
+ Task GenerateByDomainAsync(
+ string domain,
+ PdfProfile profile,
+ bool branding,
+ CancellationToken cancellationToken = default);
+}
+
+public sealed class PdfReportService : IPdfReportService
+{
+ private readonly IReportDataClient _client;
+ private readonly IAppSettingsClient _brandingClient;
+ private readonly AuditPdfGenerator _generator;
+ private readonly ILogger _logger;
+
+ public PdfReportService(
+ IReportDataClient client,
+ IAppSettingsClient brandingClient,
+ AuditPdfGenerator generator,
+ ILogger logger)
+ {
+ _client = client;
+ _brandingClient = brandingClient;
+ _generator = generator;
+ _logger = logger;
+ }
+
+ public async Task GenerateByReportIdAsync(
+ int reportId,
+ PdfProfile profile,
+ bool branding,
+ CancellationToken cancellationToken = default)
+ {
+ var payload = await _client.GetPayloadAsync(reportId, cancellationToken);
+ if (payload is null)
+ {
+ throw new KeyNotFoundException($"Report {reportId} not found");
+ }
+ var brand = await _brandingClient.GetBrandingAsync(branding, cancellationToken);
+ var model = AuditReportMapper.Map(payload.Value, reportId, profile, brand).WithTableOfContents(profile);
+ return _generator.Generate(model, profile);
+ }
+
+ public async Task GenerateByDomainAsync(
+ string domain,
+ PdfProfile profile,
+ bool branding,
+ CancellationToken cancellationToken = default)
+ {
+ var reports = await _client.ListReportsAsync(cancellationToken);
+ var reportId = DomainResolver.ResolveReportId(reports, domain);
+ if (reportId is null)
+ {
+ throw new KeyNotFoundException($"No report found for domain '{domain}'");
+ }
+ return await GenerateByReportIdAsync(reportId.Value, profile, branding, cancellationToken);
+ }
+}
diff --git a/services/FileService/src/FileService.Application/Services/WorkbookReportService.cs b/services/FileService/src/FileService.Application/Services/WorkbookReportService.cs
new file mode 100644
index 00000000..c5400b93
--- /dev/null
+++ b/services/FileService/src/FileService.Application/Services/WorkbookReportService.cs
@@ -0,0 +1,55 @@
+using System.Text.Json;
+using FileService.Application.Clients;
+using FileService.Application.Domain;
+using FileService.Rendering;
+using Microsoft.Extensions.Logging;
+
+namespace FileService.Application.Services;
+
+public interface IWorkbookReportService
+{
+ Task GenerateByReportIdAsync(int reportId, CancellationToken cancellationToken = default);
+
+ Task GenerateByDomainAsync(string domain, CancellationToken cancellationToken = default);
+}
+
+public sealed class WorkbookReportService : IWorkbookReportService
+{
+ private readonly IReportDataClient _client;
+ private readonly AuditWorkbookGenerator _generator;
+ private readonly ILogger _logger;
+
+ public WorkbookReportService(
+ IReportDataClient client,
+ AuditWorkbookGenerator generator,
+ ILogger logger)
+ {
+ _client = client;
+ _generator = generator;
+ _logger = logger;
+ }
+
+ public async Task GenerateByReportIdAsync(int reportId, CancellationToken cancellationToken = default)
+ {
+ var payload = await _client.GetPayloadAsync(reportId, cancellationToken);
+ if (payload is null)
+ {
+ throw new KeyNotFoundException($"Report {reportId} not found");
+ }
+
+ _logger.LogDebug("Generating workbook for report {ReportId}", reportId);
+ return _generator.Generate(payload.Value);
+ }
+
+ public async Task GenerateByDomainAsync(string domain, CancellationToken cancellationToken = default)
+ {
+ var reports = await _client.ListReportsAsync(cancellationToken);
+ var reportId = DomainResolver.ResolveReportId(reports, domain);
+ if (reportId is null)
+ {
+ throw new KeyNotFoundException($"No report found for domain '{domain}'");
+ }
+
+ return await GenerateByReportIdAsync(reportId.Value, cancellationToken);
+ }
+}
diff --git a/services/FileService/src/FileService.Domain/FileService.Domain.csproj b/services/FileService/src/FileService.Domain/FileService.Domain.csproj
new file mode 100644
index 00000000..b7601447
--- /dev/null
+++ b/services/FileService/src/FileService.Domain/FileService.Domain.csproj
@@ -0,0 +1,9 @@
+
+
+
+ net10.0
+ enable
+ enable
+
+
+
diff --git a/services/FileService/src/FileService.Domain/Models/AuditReportModel.cs b/services/FileService/src/FileService.Domain/Models/AuditReportModel.cs
new file mode 100644
index 00000000..85edb70a
--- /dev/null
+++ b/services/FileService/src/FileService.Domain/Models/AuditReportModel.cs
@@ -0,0 +1,90 @@
+namespace FileService.Domain.Models;
+
+public enum PdfProfile
+{
+ Executive,
+ Standard,
+ Full,
+ Premium,
+}
+
+public sealed class AuditReportModel
+{
+ public int ReportId { get; init; }
+ public string SiteName { get; init; } = "Site";
+ public string ReportTitle { get; init; } = "Technical SEO Audit Report";
+ public string GeneratedAt { get; init; } = "";
+ public string ExportedAt { get; init; } = "";
+ public int? HealthScore { get; init; }
+ public string ScoreBand { get; init; } = "";
+ public int TotalIssueCount { get; init; }
+ public IReadOnlyList DataSources { get; init; } = [];
+ public PdfBrandingModel Branding { get; init; } = new();
+ public ExecutiveSummaryModel ExecutiveSummary { get; init; } = new();
+ public IReadOnlyList CategoryScores { get; init; } = [];
+ public IReadOnlyList Issues { get; init; } = [];
+ public IReadOnlyDictionary IssueCounts { get; init; } = new Dictionary();
+ public AuditSnapshotModel? Snapshot { get; init; }
+ public LighthouseChapterModel? Lighthouse { get; init; }
+ public SearchVisibilityModel? SearchVisibility { get; init; }
+ public TrafficSnapshotModel? Traffic { get; init; }
+ public SecurityChapterModel? Security { get; init; }
+ public ContentChapterModel? Content { get; init; }
+ public IndexationChapterModel? Indexation { get; init; }
+ public IReadOnlyList LinkSamples { get; init; } = [];
+ public IReadOnlyList TruncationNotes { get; init; } = [];
+ public CrawlScopeModel? CrawlScope { get; init; }
+ public IReadOnlyList TableOfContents { get; init; } = [];
+}
+
+public sealed class ExecutiveSummaryModel
+{
+ public string Summary { get; init; } = "";
+ public string SourceLabel { get; init; } = "";
+ public IReadOnlyList Priorities { get; init; } = [];
+ public IReadOnlyList TopIssues { get; init; } = [];
+}
+
+public sealed class CategoryScoreModel
+{
+ public string Name { get; init; } = "";
+ public int? Score { get; init; }
+ public int IssueCount { get; init; }
+}
+
+public sealed class IssueModel
+{
+ public string Category { get; init; } = "";
+ public string Priority { get; init; } = "";
+ public string Message { get; init; } = "";
+ public string Headline { get; init; } = "";
+ public string Url { get; init; } = "";
+ public string UrlPath { get; init; } = "";
+ public string Recommendation { get; init; } = "";
+ public int? GscClicks { get; init; }
+ public int? GscImpressions { get; init; }
+ public int? ImpactScore { get; init; }
+}
+
+public sealed class LighthouseSummaryModel
+{
+ public string Url { get; init; } = "";
+ public int? Performance { get; init; }
+ public int? Accessibility { get; init; }
+ public int? BestPractices { get; init; }
+ public int? Seo { get; init; }
+}
+
+public sealed class CrawlScopeModel
+{
+ public int? PagesCrawled { get; init; }
+ public int? MaxPagesConfigured { get; init; }
+}
+
+public sealed class ReportListRow
+{
+ public int Id { get; init; }
+ public string? CanonicalDomain { get; init; }
+ public string? SiteName { get; init; }
+ public string? GeneratedAt { get; init; }
+}
diff --git a/services/FileService/src/FileService.Domain/Models/ChapterModels.cs b/services/FileService/src/FileService.Domain/Models/ChapterModels.cs
new file mode 100644
index 00000000..626c7ef3
--- /dev/null
+++ b/services/FileService/src/FileService.Domain/Models/ChapterModels.cs
@@ -0,0 +1,94 @@
+namespace FileService.Domain.Models;
+
+public sealed class PdfBrandingModel
+{
+ public bool Enabled { get; init; }
+ public string AgencyName { get; init; } = "";
+ public string AgencySubtitle { get; init; } = "";
+ public byte[]? LogoBytes { get; init; }
+}
+
+public sealed class AuditSnapshotModel
+{
+ public int? TotalUrls { get; init; }
+ public int? IndexableUrls { get; init; }
+ public int? TotalIssues { get; init; }
+ public int? CriticalIssues { get; init; }
+ public IReadOnlyDictionary StatusCounts { get; init; } = new Dictionary();
+ public string? GoogleFetchedAt { get; init; }
+ public string? RenderMode { get; init; }
+}
+
+public sealed class LinkSampleModel
+{
+ public string Url { get; init; } = "";
+ public string Status { get; init; } = "";
+ public string Title { get; init; } = "";
+}
+
+public sealed class LighthouseChapterModel
+{
+ public LighthouseSummaryModel Summary { get; init; } = new();
+ public string HumanSummary { get; init; } = "";
+ public IReadOnlyList Diagnostics { get; init; } = [];
+}
+
+public sealed class LighthouseDiagnosticModel
+{
+ public string Title { get; init; } = "";
+ public string Description { get; init; } = "";
+}
+
+public sealed class SearchVisibilityModel
+{
+ public IReadOnlyList TopQueries { get; init; } = [];
+ public IReadOnlyList TopPages { get; init; } = [];
+}
+
+public sealed class TrafficSnapshotModel
+{
+ public IReadOnlyList Channels { get; init; } = [];
+ public IReadOnlyList Devices { get; init; } = [];
+}
+
+public sealed class MetricRowModel
+{
+ public string Label { get; init; } = "";
+ public string Value { get; init; } = "";
+ public string? Secondary { get; init; }
+}
+
+public sealed class SecurityChapterModel
+{
+ public IReadOnlyList Findings { get; init; } = [];
+}
+
+public sealed class SecurityFindingModel
+{
+ public string Severity { get; init; } = "";
+ public string Type { get; init; } = "";
+ public string Url { get; init; } = "";
+ public string Message { get; init; } = "";
+}
+
+public sealed class ContentChapterModel
+{
+ public int? MeanWordCount { get; init; }
+ public int? MedianWordCount { get; init; }
+ public int? ThinContentCount { get; init; }
+ public IReadOnlyList TopKeywords { get; init; } = [];
+}
+
+public sealed class IndexationChapterModel
+{
+ public int? Indexable { get; init; }
+ public int? NonIndexable { get; init; }
+ public int? Blocked { get; init; }
+ public string? Notes { get; init; }
+}
+
+public sealed class TocEntryModel
+{
+ public string Title { get; init; } = "";
+ public PdfSectionId SectionId { get; init; }
+}
diff --git a/services/FileService/src/FileService.Domain/Models/PdfSectionId.cs b/services/FileService/src/FileService.Domain/Models/PdfSectionId.cs
new file mode 100644
index 00000000..7e992f16
--- /dev/null
+++ b/services/FileService/src/FileService.Domain/Models/PdfSectionId.cs
@@ -0,0 +1,18 @@
+namespace FileService.Domain.Models;
+
+public enum PdfSectionId
+{
+ Cover,
+ TableOfContents,
+ ScoreDashboard,
+ AuditSnapshot,
+ ExecutiveNarrative,
+ Findings,
+ Lighthouse,
+ SearchVisibility,
+ TrafficSnapshot,
+ Security,
+ Content,
+ Indexation,
+ Appendix,
+}
diff --git a/services/FileService/src/FileService.Rendering/AuditPdfGenerator.cs b/services/FileService/src/FileService.Rendering/AuditPdfGenerator.cs
new file mode 100644
index 00000000..d572dfb6
--- /dev/null
+++ b/services/FileService/src/FileService.Rendering/AuditPdfGenerator.cs
@@ -0,0 +1,66 @@
+using FileService.Domain.Models;
+using FileService.Rendering.Templates;
+using QuestPDF.Fluent;
+using QuestPDF.Helpers;
+using QuestPDF.Infrastructure;
+
+namespace FileService.Rendering;
+
+public sealed class AuditPdfGenerator
+{
+ public AuditPdfGenerator()
+ {
+ QuestPDF.Settings.License = LicenseType.Community;
+ }
+
+ public byte[] Generate(AuditReportModel model, PdfProfile profile)
+ {
+ var document = Document.Create(container =>
+ {
+ container.Page(page =>
+ {
+ page.Size(PageSizes.A4);
+ page.Margin(0);
+ page.Content().Element(c => CoverPageTemplate.Compose(c, model, profile));
+ });
+
+ var ctx = new Composition.PdfRenderContext { Model = model, Profile = profile };
+ var index = 0;
+ foreach (var composer in Composition.SectionRegistry.GetComposers(profile, model))
+ {
+ index++;
+ ctx.SectionIndex = index;
+ var showDivider = profile is PdfProfile.Premium or PdfProfile.Full;
+ container.Page(page =>
+ {
+ page.Size(PageSizes.A4);
+ page.MarginHorizontal(PdfTheme.Margin);
+ page.MarginTop(32);
+ page.MarginBottom(36);
+ page.DefaultTextStyle(x => x.FontSize(PdfTheme.BodySize).FontColor(PdfTheme.TextColor));
+ page.Header().Height(28).Element(c => PdfTheme.ComposeContentHeader(c, model));
+ page.Content().PaddingTop(8).Column(column =>
+ {
+ if (showDivider)
+ {
+ column.Item().Element(c => PdfTheme.SectionDivider(c, index, composer.Title));
+ }
+ column.Item().Element(c => composer.Compose(c, ctx));
+ });
+ page.Footer().Height(24).Element(c =>
+ {
+ c.AlignMiddle().Text(text =>
+ {
+ text.DefaultTextStyle(x => x.FontSize(8).FontColor(PdfTheme.MutedColor));
+ text.Span($"{model.SiteName} · ");
+ text.Span($"Exported {model.ExportedAt} · Page ");
+ text.CurrentPageNumber();
+ });
+ });
+ });
+ }
+ });
+
+ return document.GeneratePdf();
+ }
+}
diff --git a/services/FileService/src/FileService.Rendering/AuditWorkbookGenerator.cs b/services/FileService/src/FileService.Rendering/AuditWorkbookGenerator.cs
new file mode 100644
index 00000000..8fab46f5
--- /dev/null
+++ b/services/FileService/src/FileService.Rendering/AuditWorkbookGenerator.cs
@@ -0,0 +1,288 @@
+using System.Text.Json;
+using ClosedXML.Excel;
+
+namespace FileService.Rendering;
+
+public sealed class AuditWorkbookGenerator
+{
+ private static readonly string[] UrlColumns =
+ [
+ "url", "status", "title", "meta_description", "h1",
+ "canonical_url", "inlinks", "outlinks", "depth", "word_count",
+ ];
+
+ private static readonly string[] EdgeColumns =
+ [
+ "from_url", "to_url", "anchor_text", "rel",
+ "is_nofollow", "is_sponsored", "is_ugc", "link_type", "position",
+ ];
+
+ private static readonly string[] RedirectColumns =
+ [
+ "url", "message", "priority", "recommendation",
+ ];
+
+ private static readonly string[] IssueColumns =
+ [
+ "category", "priority", "message", "url",
+ "impact_score", "gsc_clicks", "gsc_impressions", "ga4_sessions",
+ "recommendation",
+ ];
+
+ public byte[] Generate(JsonElement payload)
+ {
+ using var workbook = new XLWorkbook();
+
+ WriteSheetFromObjects(workbook, "Internal URLs", GetArray(payload, "links"), UrlColumns);
+ WriteSheetFromObjects(workbook, "Links", GetArray(payload, "link_edges"), EdgeColumns);
+ WriteSheetFromObjects(workbook, "Redirects", GetArray(payload, "redirects"), RedirectColumns);
+ WriteIssueSheet(workbook, payload);
+ WriteCustomFieldsSheet(workbook, payload);
+
+ using var stream = new MemoryStream();
+ workbook.SaveAs(stream);
+ return stream.ToArray();
+ }
+
+ private static void WriteIssueSheet(XLWorkbook workbook, JsonElement payload)
+ {
+ var rows = new List>();
+ if (payload.TryGetProperty("categories", out var categories) && categories.ValueKind == JsonValueKind.Array)
+ {
+ foreach (var cat in categories.EnumerateArray())
+ {
+ var catName = GetString(cat, "name") ?? GetString(cat, "id") ?? "";
+ if (!cat.TryGetProperty("issues", out var issues) || issues.ValueKind != JsonValueKind.Array)
+ {
+ continue;
+ }
+
+ foreach (var issue in issues.EnumerateArray())
+ {
+ if (issue.ValueKind != JsonValueKind.Object)
+ {
+ continue;
+ }
+
+ var row = ObjectToRow(issue);
+ row["category"] = catName;
+ rows.Add(row);
+ }
+ }
+ }
+
+ WriteSheetFromRows(workbook, "Issues", rows, IssueColumns);
+ }
+
+ private static void WriteCustomFieldsSheet(XLWorkbook workbook, JsonElement payload)
+ {
+ var links = GetArray(payload, "links");
+ if (links.Count == 0)
+ {
+ return;
+ }
+
+ var fieldNames = new SortedSet(StringComparer.Ordinal);
+ var rows = new List>();
+
+ foreach (var link in links)
+ {
+ if (link.ValueKind != JsonValueKind.Object)
+ {
+ continue;
+ }
+
+ var url = GetString(link, "url");
+ var customExtract = GetString(link, "custom_extract");
+ var fields = ParseCustomFields(link);
+ if (string.IsNullOrWhiteSpace(url) || (string.IsNullOrWhiteSpace(customExtract) && fields.Count == 0))
+ {
+ continue;
+ }
+
+ foreach (var key in fields.Keys)
+ {
+ fieldNames.Add(key);
+ }
+
+ var row = new Dictionary(StringComparer.Ordinal)
+ {
+ ["url"] = url,
+ ["custom_extract"] = customExtract ?? "",
+ };
+ foreach (var (key, value) in fields)
+ {
+ row[key] = value;
+ }
+
+ rows.Add(row);
+ }
+
+ if (rows.Count == 0)
+ {
+ return;
+ }
+
+ var columns = new List { "url", "custom_extract" };
+ columns.AddRange(fieldNames);
+ WriteSheetFromRows(workbook, "Custom Fields", rows, columns);
+ }
+
+ private static List GetArray(JsonElement payload, string name)
+ {
+ var rows = new List();
+ if (!payload.TryGetProperty(name, out var el) || el.ValueKind != JsonValueKind.Array)
+ {
+ return rows;
+ }
+
+ foreach (var item in el.EnumerateArray())
+ {
+ rows.Add(item);
+ }
+
+ return rows;
+ }
+
+ private static void WriteSheetFromObjects(
+ XLWorkbook workbook,
+ string sheetName,
+ IReadOnlyList objects,
+ IReadOnlyList columns)
+ {
+ if (objects.Count == 0)
+ {
+ return;
+ }
+
+ var rows = objects
+ .Where(o => o.ValueKind == JsonValueKind.Object)
+ .Select(ObjectToRow)
+ .ToList();
+ WriteSheetFromRows(workbook, sheetName, rows, columns);
+ }
+
+ private static void WriteSheetFromRows(
+ XLWorkbook workbook,
+ string sheetName,
+ IReadOnlyList> rows,
+ IReadOnlyList columns)
+ {
+ if (rows.Count == 0)
+ {
+ return;
+ }
+
+ var sheet = workbook.Worksheets.Add(sheetName);
+ for (var c = 0; c < columns.Count; c++)
+ {
+ sheet.Cell(1, c + 1).Value = columns[c];
+ }
+
+ sheet.Row(1).Style.Font.Bold = true;
+
+ for (var r = 0; r < rows.Count; r++)
+ {
+ var row = rows[r];
+ for (var c = 0; c < columns.Count; c++)
+ {
+ var key = columns[c];
+ row.TryGetValue(key, out var value);
+ sheet.Cell(r + 2, c + 1).Value = value switch
+ {
+ null => Blank.Value,
+ bool b => b,
+ int i => i,
+ long l => l,
+ double d => d,
+ _ => value.ToString() ?? "",
+ };
+ }
+ }
+
+ sheet.Columns().AdjustToContents();
+ }
+
+ private static Dictionary ObjectToRow(JsonElement obj)
+ {
+ var row = new Dictionary(StringComparer.Ordinal);
+ foreach (var prop in obj.EnumerateObject())
+ {
+ row[prop.Name] = JsonValue(prop.Value);
+ }
+
+ return row;
+ }
+
+ private static object? JsonValue(JsonElement el) => el.ValueKind switch
+ {
+ JsonValueKind.Null or JsonValueKind.Undefined => null,
+ JsonValueKind.True => true,
+ JsonValueKind.False => false,
+ JsonValueKind.Number when el.TryGetInt64(out var l) => l,
+ JsonValueKind.Number => el.GetDouble(),
+ JsonValueKind.String => el.GetString(),
+ _ => el.ToString(),
+ };
+
+ private static Dictionary ParseCustomFields(JsonElement link)
+ {
+ if (!link.TryGetProperty("custom_fields", out var raw))
+ {
+ return new Dictionary(StringComparer.Ordinal);
+ }
+
+ if (raw.ValueKind == JsonValueKind.Object)
+ {
+ var dict = new Dictionary(StringComparer.Ordinal);
+ foreach (var prop in raw.EnumerateObject())
+ {
+ dict[prop.Name] = prop.Value.ToString();
+ }
+
+ return dict;
+ }
+
+ if (raw.ValueKind != JsonValueKind.String)
+ {
+ return new Dictionary(StringComparer.Ordinal);
+ }
+
+ var text = raw.GetString()?.Trim();
+ if (string.IsNullOrEmpty(text))
+ {
+ return new Dictionary(StringComparer.Ordinal);
+ }
+
+ try
+ {
+ using var doc = JsonDocument.Parse(text);
+ if (doc.RootElement.ValueKind != JsonValueKind.Object)
+ {
+ return new Dictionary(StringComparer.Ordinal);
+ }
+
+ var parsed = new Dictionary(StringComparer.Ordinal);
+ foreach (var prop in doc.RootElement.EnumerateObject())
+ {
+ parsed[prop.Name] = prop.Value.ToString();
+ }
+
+ return parsed;
+ }
+ catch (JsonException)
+ {
+ return new Dictionary(StringComparer.Ordinal);
+ }
+ }
+
+ private static string? GetString(JsonElement el, string name)
+ {
+ if (!el.TryGetProperty(name, out var prop) || prop.ValueKind == JsonValueKind.Null)
+ {
+ return null;
+ }
+
+ return prop.GetString();
+ }
+}
diff --git a/services/FileService/src/FileService.Rendering/Charts/HorizontalBarChart.cs b/services/FileService/src/FileService.Rendering/Charts/HorizontalBarChart.cs
new file mode 100644
index 00000000..10d7b709
--- /dev/null
+++ b/services/FileService/src/FileService.Rendering/Charts/HorizontalBarChart.cs
@@ -0,0 +1,24 @@
+using QuestPDF.Fluent;
+using QuestPDF.Infrastructure;
+
+namespace FileService.Rendering.Charts;
+
+public static class HorizontalBarChart
+{
+ public static void Compose(IContainer container, int value, int max = 100, string? barColor = null)
+ {
+ var pct = max <= 0 ? 0 : Math.Clamp(value / (double)max, 0, 1);
+ var color = barColor ?? PdfTheme.AccentColor;
+ container.Height(8).Background(PdfTheme.MutedBg).Row(row =>
+ {
+ if (pct > 0)
+ {
+ row.RelativeItem((float)pct).Background(color);
+ }
+ if (pct < 1)
+ {
+ row.RelativeItem((float)(1 - pct));
+ }
+ });
+ }
+}
diff --git a/services/FileService/src/FileService.Rendering/Composition/PdfRenderContext.cs b/services/FileService/src/FileService.Rendering/Composition/PdfRenderContext.cs
new file mode 100644
index 00000000..c728fc3d
--- /dev/null
+++ b/services/FileService/src/FileService.Rendering/Composition/PdfRenderContext.cs
@@ -0,0 +1,19 @@
+using FileService.Domain.Models;
+using QuestPDF.Infrastructure;
+
+namespace FileService.Rendering.Composition;
+
+public sealed class PdfRenderContext
+{
+ public required AuditReportModel Model { get; init; }
+ public PdfProfile Profile { get; init; }
+ public int SectionIndex { get; set; }
+}
+
+public interface IPdfSectionComposer
+{
+ PdfSectionId SectionId { get; }
+ string Title { get; }
+ bool IsAvailable(PdfRenderContext context);
+ void Compose(IContainer container, PdfRenderContext context);
+}
diff --git a/services/FileService/src/FileService.Rendering/Composition/SectionRegistry.cs b/services/FileService/src/FileService.Rendering/Composition/SectionRegistry.cs
new file mode 100644
index 00000000..54a203fc
--- /dev/null
+++ b/services/FileService/src/FileService.Rendering/Composition/SectionRegistry.cs
@@ -0,0 +1,89 @@
+using FileService.Domain.Models;
+using FileService.Rendering.Sections;
+
+namespace FileService.Rendering.Composition;
+
+public static class SectionRegistry
+{
+ private static readonly IPdfSectionComposer[] AllComposers =
+ [
+ new TableOfContentsSection(),
+ new ScoreDashboardSection(),
+ new AuditSnapshotSection(),
+ new ExecutiveNarrativeSection(),
+ new FindingsSection(),
+ new LighthouseChapterSection(),
+ new SearchVisibilitySection(),
+ new TrafficSnapshotSection(),
+ new SecurityChapterSection(),
+ new ContentChapterSection(),
+ new IndexationChapterSection(),
+ new AppendixSection(),
+ ];
+
+ private static readonly Dictionary ById =
+ AllComposers.ToDictionary(c => c.SectionId);
+
+ public static IReadOnlyList ResolveSectionOrder(PdfProfile profile, AuditReportModel model)
+ {
+ var ids = profile switch
+ {
+ PdfProfile.Executive => new[]
+ {
+ PdfSectionId.ScoreDashboard,
+ PdfSectionId.ExecutiveNarrative,
+ PdfSectionId.Findings,
+ },
+ PdfProfile.Standard => new[]
+ {
+ PdfSectionId.TableOfContents,
+ PdfSectionId.ScoreDashboard,
+ PdfSectionId.AuditSnapshot,
+ PdfSectionId.Findings,
+ PdfSectionId.Appendix,
+ },
+ PdfProfile.Full => new[]
+ {
+ PdfSectionId.TableOfContents,
+ PdfSectionId.ScoreDashboard,
+ PdfSectionId.AuditSnapshot,
+ PdfSectionId.ExecutiveNarrative,
+ PdfSectionId.Findings,
+ PdfSectionId.Lighthouse,
+ PdfSectionId.SearchVisibility,
+ PdfSectionId.TrafficSnapshot,
+ PdfSectionId.Security,
+ PdfSectionId.Content,
+ PdfSectionId.Indexation,
+ PdfSectionId.Appendix,
+ },
+ PdfProfile.Premium => new[]
+ {
+ PdfSectionId.TableOfContents,
+ PdfSectionId.ScoreDashboard,
+ PdfSectionId.ExecutiveNarrative,
+ PdfSectionId.Findings,
+ PdfSectionId.Lighthouse,
+ PdfSectionId.SearchVisibility,
+ PdfSectionId.TrafficSnapshot,
+ PdfSectionId.Security,
+ PdfSectionId.Content,
+ PdfSectionId.Indexation,
+ PdfSectionId.Appendix,
+ },
+ _ => new[] { PdfSectionId.Findings, PdfSectionId.Appendix },
+ };
+
+ return ids.Where(id => ById[id].IsAvailable(new PdfRenderContext { Model = model, Profile = profile })).ToList();
+ }
+
+ public static IPdfSectionComposer Get(PdfSectionId id) => ById[id];
+
+ public static IEnumerable GetComposers(PdfProfile profile, AuditReportModel model)
+ {
+ foreach (var id in ResolveSectionOrder(profile, model))
+ {
+ yield return ById[id];
+ }
+ }
+}
diff --git a/services/FileService/src/FileService.Rendering/Composition/TocBuilder.cs b/services/FileService/src/FileService.Rendering/Composition/TocBuilder.cs
new file mode 100644
index 00000000..0cae2eaa
--- /dev/null
+++ b/services/FileService/src/FileService.Rendering/Composition/TocBuilder.cs
@@ -0,0 +1,72 @@
+using FileService.Domain.Models;
+using FileService.Rendering.Composition;
+
+namespace FileService.Rendering.Composition;
+
+public static class TocBuilder
+{
+ private static readonly Dictionary Titles = new()
+ {
+ [PdfSectionId.TableOfContents] = "Contents",
+ [PdfSectionId.ScoreDashboard] = "Score overview",
+ [PdfSectionId.AuditSnapshot] = "Audit snapshot",
+ [PdfSectionId.ExecutiveNarrative] = "Executive summary",
+ [PdfSectionId.Findings] = "Findings",
+ [PdfSectionId.Lighthouse] = "Lighthouse",
+ [PdfSectionId.SearchVisibility] = "Search visibility",
+ [PdfSectionId.TrafficSnapshot] = "Traffic snapshot",
+ [PdfSectionId.Security] = "Security",
+ [PdfSectionId.Content] = "Content quality",
+ [PdfSectionId.Indexation] = "Indexation",
+ [PdfSectionId.Appendix] = "Appendix",
+ };
+
+ public static IReadOnlyList Build(AuditReportModel model, PdfProfile profile)
+ {
+ var order = SectionRegistry.ResolveSectionOrder(profile, model);
+ return order
+ .Where(id => id != PdfSectionId.TableOfContents)
+ .Select(id => new TocEntryModel { SectionId = id, Title = Titles.GetValueOrDefault(id, id.ToString()) })
+ .ToList();
+ }
+}
+
+public static class AuditReportModelExtensions
+{
+ public static AuditReportModel WithTableOfContents(this AuditReportModel model, PdfProfile profile)
+ {
+ var toc = TocBuilder.Build(model, profile);
+ if (toc.Count == 0)
+ {
+ return model;
+ }
+ return new AuditReportModel
+ {
+ ReportId = model.ReportId,
+ SiteName = model.SiteName,
+ ReportTitle = model.ReportTitle,
+ GeneratedAt = model.GeneratedAt,
+ ExportedAt = model.ExportedAt,
+ HealthScore = model.HealthScore,
+ ScoreBand = model.ScoreBand,
+ TotalIssueCount = model.TotalIssueCount,
+ DataSources = model.DataSources,
+ Branding = model.Branding,
+ ExecutiveSummary = model.ExecutiveSummary,
+ CategoryScores = model.CategoryScores,
+ Issues = model.Issues,
+ IssueCounts = model.IssueCounts,
+ Snapshot = model.Snapshot,
+ Lighthouse = model.Lighthouse,
+ SearchVisibility = model.SearchVisibility,
+ Traffic = model.Traffic,
+ Security = model.Security,
+ Content = model.Content,
+ Indexation = model.Indexation,
+ LinkSamples = model.LinkSamples,
+ TruncationNotes = model.TruncationNotes,
+ CrawlScope = model.CrawlScope,
+ TableOfContents = toc,
+ };
+ }
+}
diff --git a/services/FileService/src/FileService.Rendering/FileService.Rendering.csproj b/services/FileService/src/FileService.Rendering/FileService.Rendering.csproj
new file mode 100644
index 00000000..f77c6c00
--- /dev/null
+++ b/services/FileService/src/FileService.Rendering/FileService.Rendering.csproj
@@ -0,0 +1,18 @@
+
+
+
+
+
+
+
+
+
+
+
+
+ net10.0
+ enable
+ enable
+
+
+
diff --git a/services/FileService/src/FileService.Rendering/PdfTheme.cs b/services/FileService/src/FileService.Rendering/PdfTheme.cs
new file mode 100644
index 00000000..7dfba382
--- /dev/null
+++ b/services/FileService/src/FileService.Rendering/PdfTheme.cs
@@ -0,0 +1,101 @@
+using FileService.Domain.Models;
+using QuestPDF.Fluent;
+using QuestPDF.Helpers;
+using QuestPDF.Infrastructure;
+
+namespace FileService.Rendering;
+
+public static class PdfTheme
+{
+ public const float Margin = 40;
+ public const float BodySize = 10;
+ public const float SectionSize = 14;
+ public const float CoverTitleSize = 28;
+ public const string TextColor = "#1e293b";
+ public const string MutedColor = "#64748b";
+ public const string MutedBg = "#f1f5f9";
+ public const string AccentColor = "#2563eb";
+ public const string DividerColor = "#1e40af";
+ public const string CoverBandColor = "#1e3a8a";
+
+ public static readonly Dictionary PriorityColors = new(StringComparer.OrdinalIgnoreCase)
+ {
+ ["critical"] = "#dc2626",
+ ["high"] = "#ea580c",
+ ["medium"] = "#ca8a04",
+ ["low"] = "#64748b",
+ };
+
+ public static string ScoreColor(int? score) => score switch
+ {
+ >= 80 => "#16a34a",
+ >= 60 => "#ca8a04",
+ _ => "#dc2626",
+ };
+
+ public static void ComposeContentFooter(IContainer container, AuditReportModel model, int pageNumber)
+ {
+ container.Row(row =>
+ {
+ row.RelativeItem().AlignLeft().Text(model.SiteName).FontSize(8).FontColor(MutedColor);
+ row.RelativeItem().AlignCenter().Text(text =>
+ {
+ text.DefaultTextStyle(x => x.FontSize(8).FontColor(MutedColor));
+ text.Span("Confidential — prepared for client review. ");
+ text.Span($"Exported {model.ExportedAt}");
+ });
+ row.RelativeItem().AlignRight().Text($"Page {pageNumber}").FontSize(8).FontColor(MutedColor);
+ });
+ }
+
+ public static void ComposeContentHeader(IContainer container, AuditReportModel model)
+ {
+ container.Row(row =>
+ {
+ if (model.Branding.Enabled && model.Branding.LogoBytes is { Length: > 0 })
+ {
+ row.ConstantItem(48).Height(24).Image(model.Branding.LogoBytes).FitArea();
+ }
+ else if (model.Branding.Enabled && !string.IsNullOrWhiteSpace(model.Branding.AgencyName))
+ {
+ row.AutoItem().Text(model.Branding.AgencyName).FontSize(9).Bold().FontColor(MutedColor);
+ }
+ row.RelativeItem().AlignRight().Text(model.ReportTitle).FontSize(8).FontColor(MutedColor);
+ });
+ }
+
+ public static void SectionTitle(IContainer container, string title)
+ {
+ container.PaddingTop(12).PaddingBottom(6).Text(title)
+ .FontSize(SectionSize)
+ .Bold()
+ .FontColor(TextColor);
+ }
+
+ public static void SectionDivider(IContainer container, int chapterNumber, string title)
+ {
+ container.PaddingVertical(16).Background(CoverBandColor).Padding(20).Row(row =>
+ {
+ row.AutoItem().Text($"{chapterNumber:D2}").FontSize(24).Bold().FontColor(Colors.White);
+ row.RelativeItem().PaddingLeft(12).AlignMiddle().Text(title).FontSize(18).Bold().FontColor(Colors.White);
+ });
+ }
+
+ public static void PriorityBadge(IContainer container, string priority)
+ {
+ var color = PriorityColors.GetValueOrDefault(priority.ToLowerInvariant(), MutedColor);
+ container
+ .Background(color)
+ .PaddingHorizontal(6)
+ .PaddingVertical(2)
+ .Text(priority.ToUpperInvariant())
+ .FontSize(8)
+ .Bold()
+ .FontColor(Colors.White);
+ }
+
+ public static void CalloutBox(IContainer container, Action content)
+ {
+ container.Background(MutedBg).BorderLeft(4).BorderColor(AccentColor).Padding(12).Column(content);
+ }
+}
diff --git a/services/FileService/src/FileService.Rendering/Sections/PdfSections.cs b/services/FileService/src/FileService.Rendering/Sections/PdfSections.cs
new file mode 100644
index 00000000..675b9999
--- /dev/null
+++ b/services/FileService/src/FileService.Rendering/Sections/PdfSections.cs
@@ -0,0 +1,620 @@
+using FileService.Domain.Models;
+using FileService.Rendering.Charts;
+using FileService.Rendering.Composition;
+using QuestPDF.Fluent;
+using QuestPDF.Infrastructure;
+
+namespace FileService.Rendering.Sections;
+
+public sealed class TableOfContentsSection : IPdfSectionComposer
+{
+ public PdfSectionId SectionId => PdfSectionId.TableOfContents;
+ public string Title => "Contents";
+
+ public bool IsAvailable(PdfRenderContext context) =>
+ context.Model.TableOfContents.Count > 0;
+
+ public void Compose(IContainer container, PdfRenderContext context)
+ {
+ container.Column(column =>
+ {
+ column.Item().Element(c => PdfTheme.SectionTitle(c, "Table of contents"));
+ var n = 1;
+ foreach (var entry in context.Model.TableOfContents)
+ {
+ var num = n++;
+ column.Item().PaddingVertical(4).Row(row =>
+ {
+ row.AutoItem().Width(24).Text($"{num}.").FontSize(PdfTheme.BodySize);
+ row.RelativeItem().Text(entry.Title).FontSize(PdfTheme.BodySize);
+ });
+ }
+ });
+ }
+}
+
+public sealed class ScoreDashboardSection : IPdfSectionComposer
+{
+ public PdfSectionId SectionId => PdfSectionId.ScoreDashboard;
+ public string Title => "Score overview";
+
+ public bool IsAvailable(PdfRenderContext context) => context.Model.CategoryScores.Count > 0;
+
+ public void Compose(IContainer container, PdfRenderContext context)
+ {
+ var model = context.Model;
+ container.Column(column =>
+ {
+ if (context.Profile is not PdfProfile.Premium and not PdfProfile.Full)
+ {
+ column.Item().Element(c => PdfTheme.SectionTitle(c, Title));
+ }
+
+ foreach (var chunk in model.CategoryScores.Chunk(2))
+ {
+ column.Item().PaddingBottom(8).Row(row =>
+ {
+ foreach (var cat in chunk)
+ {
+ row.RelativeItem().Padding(4).Border(1).BorderColor("#e2e8f0").Padding(10).Column(c =>
+ {
+ c.Item().Text(cat.Name).FontSize(9).FontColor(PdfTheme.MutedColor);
+ c.Item().Row(r =>
+ {
+ r.AutoItem().Text(cat.Score?.ToString() ?? "—").FontSize(22).Bold()
+ .FontColor(PdfTheme.ScoreColor(cat.Score));
+ if (cat.IssueCount > 0)
+ {
+ r.RelativeItem().AlignMiddle().PaddingLeft(8).Text($"{cat.IssueCount} issues")
+ .FontSize(8).FontColor(PdfTheme.MutedColor);
+ }
+ });
+ if (cat.Score is not null)
+ {
+ c.Item().PaddingTop(6).Element(e => HorizontalBarChart.Compose(e, cat.Score.Value));
+ }
+ });
+ }
+ });
+ }
+
+ if (model.Snapshot is not null)
+ {
+ column.Item().PaddingTop(8).Row(row =>
+ {
+ AddKpi(row, "Pages crawled", model.CrawlScope?.PagesCrawled?.ToString() ?? "—");
+ AddKpi(row, "Total URLs", model.Snapshot.TotalUrls?.ToString() ?? "—");
+ AddKpi(row, "Indexable", model.Snapshot.IndexableUrls?.ToString() ?? "—");
+ AddKpi(row, "Total issues", model.Snapshot.TotalIssues?.ToString() ?? "—");
+ });
+ }
+ });
+ }
+
+ private static void AddKpi(RowDescriptor row, string label, string value)
+ {
+ row.RelativeItem().Padding(4).Background(PdfTheme.MutedBg).Padding(8).Column(c =>
+ {
+ c.Item().Text(label).FontSize(8).FontColor(PdfTheme.MutedColor);
+ c.Item().Text(value).FontSize(14).Bold();
+ });
+ }
+}
+
+public sealed class AuditSnapshotSection : IPdfSectionComposer
+{
+ public PdfSectionId SectionId => PdfSectionId.AuditSnapshot;
+ public string Title => "Audit snapshot";
+
+ public bool IsAvailable(PdfRenderContext context) => context.Model.Snapshot is not null;
+
+ public void Compose(IContainer container, PdfRenderContext context)
+ {
+ var snap = context.Model.Snapshot!;
+ container.Column(column =>
+ {
+ column.Item().Element(c => PdfTheme.SectionTitle(c, Title));
+ foreach (var (key, val) in SnapshotLines(context.Model, snap))
+ {
+ column.Item().PaddingVertical(3).Row(row =>
+ {
+ row.ConstantItem(140).Text(key).FontSize(9).Bold();
+ row.RelativeItem().Text(val).FontSize(9);
+ });
+ }
+ });
+ }
+
+ private static IEnumerable<(string, string)> SnapshotLines(AuditReportModel model, AuditSnapshotModel snap)
+ {
+ yield return ("Property", model.SiteName);
+ if (!string.IsNullOrWhiteSpace(model.GeneratedAt))
+ {
+ yield return ("Report generated", model.GeneratedAt);
+ }
+ if (model.DataSources.Count > 0)
+ {
+ yield return ("Data sources", string.Join(", ", model.DataSources));
+ }
+ if (model.CrawlScope?.PagesCrawled is not null)
+ {
+ var scope = $"{model.CrawlScope.PagesCrawled} pages crawled";
+ if (model.CrawlScope.MaxPagesConfigured is not null)
+ {
+ scope += $" (limit {model.CrawlScope.MaxPagesConfigured})";
+ }
+ yield return ("Crawl scope", scope);
+ }
+ if (!string.IsNullOrWhiteSpace(snap.RenderMode))
+ {
+ yield return ("Render mode", snap.RenderMode);
+ }
+ if (snap.TotalUrls is not null)
+ {
+ yield return ("URLs in crawl", snap.TotalUrls.ToString()!);
+ }
+ if (snap.IndexableUrls is not null)
+ {
+ yield return ("Indexable URLs", snap.IndexableUrls.ToString()!);
+ }
+ if (snap.StatusCounts.Count > 0)
+ {
+ yield return ("HTTP status mix", string.Join(", ", snap.StatusCounts.Select(kv => $"{kv.Key}: {kv.Value}")));
+ }
+ }
+}
+
+public sealed class ExecutiveNarrativeSection : IPdfSectionComposer
+{
+ public PdfSectionId SectionId => PdfSectionId.ExecutiveNarrative;
+ public string Title => "Executive summary";
+
+ public bool IsAvailable(PdfRenderContext context)
+ {
+ var e = context.Model.ExecutiveSummary;
+ return !string.IsNullOrWhiteSpace(e.Summary) || e.Priorities.Count > 0;
+ }
+
+ public void Compose(IContainer container, PdfRenderContext context)
+ {
+ var exec = context.Model.ExecutiveSummary;
+ container.Column(column =>
+ {
+ column.Item().Element(c => PdfTheme.SectionTitle(c, Title));
+ column.Item().Element(c => PdfTheme.CalloutBox(c, col =>
+ {
+ if (!string.IsNullOrWhiteSpace(exec.SourceLabel))
+ {
+ col.Item().Text($"Source: {exec.SourceLabel}").FontSize(8).FontColor(PdfTheme.MutedColor);
+ }
+ if (!string.IsNullOrWhiteSpace(exec.Summary))
+ {
+ col.Item().PaddingTop(4).Text(exec.Summary).FontSize(PdfTheme.BodySize).LineHeight(1.4f);
+ }
+ }));
+ if (exec.Priorities.Count > 0)
+ {
+ column.Item().PaddingTop(10).Text("Priorities").FontSize(11).Bold();
+ var i = 1;
+ foreach (var p in exec.Priorities)
+ {
+ column.Item().PaddingLeft(8).Text($"{i++}. {p}").FontSize(PdfTheme.BodySize);
+ }
+ }
+ });
+ }
+}
+
+public sealed class FindingsSection : IPdfSectionComposer
+{
+ public PdfSectionId SectionId => PdfSectionId.Findings;
+ public string Title => "Findings";
+
+ public bool IsAvailable(PdfRenderContext context)
+ {
+ if (context.Profile == PdfProfile.Executive)
+ {
+ return context.Model.ExecutiveSummary.TopIssues.Count > 0;
+ }
+ return context.Model.Issues.Count > 0;
+ }
+
+ public void Compose(IContainer container, PdfRenderContext context)
+ {
+ var issues = context.Profile == PdfProfile.Executive
+ ? context.Model.ExecutiveSummary.TopIssues
+ : context.Model.Issues;
+
+ container.Column(column =>
+ {
+ if (context.Profile is not PdfProfile.Premium and not PdfProfile.Full)
+ {
+ column.Item().Element(c => PdfTheme.SectionTitle(c, Title));
+ }
+
+ foreach (var note in context.Model.TruncationNotes)
+ {
+ column.Item().PaddingBottom(6).Text(note).FontSize(9).Italic().FontColor(PdfTheme.MutedColor);
+ }
+
+ if (context.Profile == PdfProfile.Executive)
+ {
+ foreach (var issue in issues)
+ {
+ column.Item().Element(c => ComposeIssueCard(c, issue, context.Profile));
+ }
+ return;
+ }
+
+ var groups = issues
+ .GroupBy(i => i.Priority, StringComparer.OrdinalIgnoreCase)
+ .OrderBy(g => PriorityOrder(g.Key));
+
+ foreach (var group in groups)
+ {
+ IEnumerable> subGroups;
+ if (group.Count() > 8)
+ {
+ subGroups = group.GroupBy(i => i.Category).OrderBy(g => g.Key);
+ }
+ else
+ {
+ subGroups = [new SimpleGrouping(group.Key, group)];
+ }
+
+ foreach (var sub in subGroups)
+ {
+ column.Item().PaddingTop(8).Row(row =>
+ {
+ row.AutoItem().Element(c => PdfTheme.PriorityBadge(c, group.Key));
+ row.AutoItem().PaddingLeft(8).Text(sub.Key == group.Key
+ ? $"{sub.Count()} issue(s)"
+ : $"{group.Key} — {sub.Key}: {sub.Count()}").FontSize(10).Bold();
+ });
+ foreach (var issue in sub)
+ {
+ column.Item().Element(c => ComposeIssueCard(c, issue, context.Profile));
+ }
+ }
+ }
+ });
+ }
+
+ private static void ComposeIssueCard(IContainer container, IssueModel issue, PdfProfile profile)
+ {
+ var priorityColor = PdfTheme.PriorityColors.GetValueOrDefault(issue.Priority.ToLowerInvariant(), PdfTheme.MutedColor);
+ container.PaddingVertical(4).Row(row =>
+ {
+ row.ConstantItem(4).Background(priorityColor);
+ row.RelativeItem().Background(PdfTheme.MutedBg).Padding(8).Column(col =>
+ {
+ col.Item().Text(issue.Headline.Length > 0 ? issue.Headline : issue.Message).FontSize(PdfTheme.BodySize).Bold();
+ if (!string.IsNullOrWhiteSpace(issue.Category))
+ {
+ col.Item().Text(issue.Category).FontSize(8).FontColor(PdfTheme.MutedColor);
+ }
+ if (!string.IsNullOrWhiteSpace(issue.UrlPath) || !string.IsNullOrWhiteSpace(issue.Url))
+ {
+ col.Item().Text(issue.UrlPath.Length > 0 ? issue.UrlPath : issue.Url).FontSize(8).FontColor(PdfTheme.AccentColor);
+ }
+ if (profile is PdfProfile.Premium or PdfProfile.Full &&
+ (issue.GscClicks is not null || issue.GscImpressions is not null))
+ {
+ col.Item().Text($"GSC: {issue.GscClicks ?? 0} clicks · {issue.GscImpressions ?? 0} impressions")
+ .FontSize(8).FontColor(PdfTheme.MutedColor);
+ }
+ if (!string.IsNullOrWhiteSpace(issue.Recommendation))
+ {
+ col.Item().PaddingTop(2).Text($"Fix: {issue.Recommendation}").FontSize(9).Italic();
+ }
+ });
+ });
+ }
+
+ private static int PriorityOrder(string priority) => priority.ToLowerInvariant() switch
+ {
+ "critical" => 0,
+ "high" => 1,
+ "medium" => 2,
+ "low" => 3,
+ _ => 9,
+ };
+
+ private sealed class SimpleGrouping(string key, IEnumerable items) : IGrouping
+ {
+ public string Key { get; } = key;
+ public IEnumerator GetEnumerator() => items.GetEnumerator();
+ System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator() => GetEnumerator();
+ }
+}
+
+public sealed class LighthouseChapterSection : IPdfSectionComposer
+{
+ public PdfSectionId SectionId => PdfSectionId.Lighthouse;
+ public string Title => "Lighthouse";
+
+ public bool IsAvailable(PdfRenderContext context) => context.Model.Lighthouse is not null;
+
+ public void Compose(IContainer container, PdfRenderContext context)
+ {
+ var lh = context.Model.Lighthouse!;
+ var s = lh.Summary;
+ container.Column(column =>
+ {
+ column.Item().Element(c => PdfTheme.SectionTitle(c, Title));
+ if (!string.IsNullOrWhiteSpace(s.Url))
+ {
+ column.Item().Text(s.Url).FontSize(9).FontColor(PdfTheme.MutedColor);
+ }
+ if (!string.IsNullOrWhiteSpace(lh.HumanSummary))
+ {
+ column.Item().PaddingTop(6).Text(lh.HumanSummary).FontSize(9);
+ }
+ column.Item().PaddingTop(8).Row(row =>
+ {
+ AddMetric(row, "Performance", s.Performance);
+ AddMetric(row, "Accessibility", s.Accessibility);
+ AddMetric(row, "Best practices", s.BestPractices);
+ AddMetric(row, "SEO", s.Seo);
+ });
+ if (lh.Diagnostics.Count > 0)
+ {
+ column.Item().PaddingTop(10).Text("Diagnostics").FontSize(11).Bold();
+ foreach (var d in lh.Diagnostics.Take(10))
+ {
+ column.Item().PaddingTop(4).Text(d.Title).FontSize(9).Bold();
+ if (!string.IsNullOrWhiteSpace(d.Description))
+ {
+ column.Item().Text(d.Description).FontSize(8).FontColor(PdfTheme.MutedColor);
+ }
+ }
+ }
+ });
+ }
+
+ private static void AddMetric(RowDescriptor row, string label, int? value)
+ {
+ row.RelativeItem().Border(1).BorderColor("#e2e8f0").Padding(8).Column(c =>
+ {
+ c.Item().Text(label).FontSize(8).FontColor(PdfTheme.MutedColor);
+ c.Item().Text(value?.ToString() ?? "—").FontSize(16).Bold().FontColor(PdfTheme.ScoreColor(value));
+ });
+ }
+}
+
+public sealed class SearchVisibilitySection : IPdfSectionComposer
+{
+ public PdfSectionId SectionId => PdfSectionId.SearchVisibility;
+ public string Title => "Search visibility";
+
+ public bool IsAvailable(PdfRenderContext context) =>
+ context.Model.SearchVisibility is { TopQueries.Count: > 0 } or { TopPages.Count: > 0 };
+
+ public void Compose(IContainer container, PdfRenderContext context)
+ {
+ var sv = context.Model.SearchVisibility!;
+ container.Column(column =>
+ {
+ column.Item().Element(c => PdfTheme.SectionTitle(c, Title));
+ if (sv.TopQueries.Count > 0)
+ {
+ column.Item().PaddingTop(6).Text("Top queries").FontSize(10).Bold();
+ ComposeTable(column, sv.TopQueries);
+ }
+ if (sv.TopPages.Count > 0)
+ {
+ column.Item().PaddingTop(10).Text("Top pages").FontSize(10).Bold();
+ ComposeTable(column, sv.TopPages);
+ }
+ });
+ }
+
+ private static void ComposeTable(ColumnDescriptor column, IReadOnlyList rows)
+ {
+ foreach (var row in rows)
+ {
+ column.Item().PaddingVertical(2).Row(r =>
+ {
+ r.RelativeItem(2).Text(row.Label).FontSize(8);
+ r.RelativeItem().AlignRight().Text(row.Value).FontSize(8).Bold();
+ if (!string.IsNullOrWhiteSpace(row.Secondary))
+ {
+ r.RelativeItem().AlignRight().Text(row.Secondary).FontSize(8).FontColor(PdfTheme.MutedColor);
+ }
+ });
+ }
+ }
+}
+
+public sealed class TrafficSnapshotSection : IPdfSectionComposer
+{
+ public PdfSectionId SectionId => PdfSectionId.TrafficSnapshot;
+ public string Title => "Traffic snapshot";
+
+ public bool IsAvailable(PdfRenderContext context) =>
+ context.Model.Traffic is { Channels.Count: > 0 } or { Devices.Count: > 0 };
+
+ public void Compose(IContainer container, PdfRenderContext context)
+ {
+ var t = context.Model.Traffic!;
+ container.Column(column =>
+ {
+ column.Item().Element(c => PdfTheme.SectionTitle(c, Title));
+ if (t.Channels.Count > 0)
+ {
+ column.Item().Text("Channels").FontSize(10).Bold();
+ foreach (var ch in t.Channels)
+ {
+ column.Item().PaddingVertical(3).Row(row =>
+ {
+ row.RelativeItem().Text(ch.Label).FontSize(9);
+ row.AutoItem().Text(ch.Value).FontSize(9).Bold();
+ });
+ if (int.TryParse(ch.Value.Replace(",", ""), out var sessions))
+ {
+ column.Item().Element(c => HorizontalBarChart.Compose(c, Math.Min(sessions, 100), 100));
+ }
+ }
+ }
+ });
+ }
+}
+
+public sealed class SecurityChapterSection : IPdfSectionComposer
+{
+ public PdfSectionId SectionId => PdfSectionId.Security;
+ public string Title => "Security";
+
+ public bool IsAvailable(PdfRenderContext context) =>
+ context.Model.Security is { Findings.Count: > 0 };
+
+ public void Compose(IContainer container, PdfRenderContext context)
+ {
+ container.Column(column =>
+ {
+ column.Item().Element(c => PdfTheme.SectionTitle(c, Title));
+ foreach (var f in context.Model.Security!.Findings.Take(20))
+ {
+ column.Item().PaddingVertical(4).BorderBottom(1).BorderColor("#e2e8f0").Column(c =>
+ {
+ c.Item().Row(row =>
+ {
+ row.AutoItem().Element(e => PdfTheme.PriorityBadge(e, f.Severity));
+ row.AutoItem().PaddingLeft(6).Text(f.Type).FontSize(9).Bold();
+ });
+ c.Item().Text(f.Message).FontSize(9);
+ if (!string.IsNullOrWhiteSpace(f.Url))
+ {
+ c.Item().Text(f.Url).FontSize(8).FontColor(PdfTheme.AccentColor);
+ }
+ });
+ }
+ });
+ }
+}
+
+public sealed class ContentChapterSection : IPdfSectionComposer
+{
+ public PdfSectionId SectionId => PdfSectionId.Content;
+ public string Title => "Content quality";
+
+ public bool IsAvailable(PdfRenderContext context) => context.Model.Content is not null;
+
+ public void Compose(IContainer container, PdfRenderContext context)
+ {
+ var c = context.Model.Content!;
+ container.Column(column =>
+ {
+ column.Item().Element(c => PdfTheme.SectionTitle(c, Title));
+ column.Item().Row(row =>
+ {
+ AddStat(row, "Mean words", c.MeanWordCount?.ToString() ?? "—");
+ AddStat(row, "Median words", c.MedianWordCount?.ToString() ?? "—");
+ AddStat(row, "Thin pages", c.ThinContentCount?.ToString() ?? "—");
+ });
+ if (c.TopKeywords.Count > 0)
+ {
+ column.Item().PaddingTop(10).Text("Top site keywords").FontSize(10).Bold();
+ foreach (var kw in c.TopKeywords.Take(15))
+ {
+ column.Item().Text($"{kw.Label} ({kw.Value})").FontSize(8);
+ }
+ }
+ });
+ }
+
+ private static void AddStat(RowDescriptor row, string label, string value)
+ {
+ row.RelativeItem().Padding(4).Background(PdfTheme.MutedBg).Padding(8).Column(col =>
+ {
+ col.Item().Text(label).FontSize(8).FontColor(PdfTheme.MutedColor);
+ col.Item().Text(value).FontSize(14).Bold();
+ });
+ }
+}
+
+public sealed class IndexationChapterSection : IPdfSectionComposer
+{
+ public PdfSectionId SectionId => PdfSectionId.Indexation;
+ public string Title => "Indexation";
+
+ public bool IsAvailable(PdfRenderContext context) => context.Model.Indexation is not null;
+
+ public void Compose(IContainer container, PdfRenderContext context)
+ {
+ var idx = context.Model.Indexation!;
+ container.Column(column =>
+ {
+ column.Item().Element(c => PdfTheme.SectionTitle(c, Title));
+ column.Item().Row(row =>
+ {
+ AddStat(row, "Indexable", idx.Indexable?.ToString() ?? "—");
+ AddStat(row, "Non-indexable", idx.NonIndexable?.ToString() ?? "—");
+ AddStat(row, "Blocked", idx.Blocked?.ToString() ?? "—");
+ });
+ if (!string.IsNullOrWhiteSpace(idx.Notes))
+ {
+ column.Item().PaddingTop(8).Text(idx.Notes).FontSize(9);
+ }
+ });
+ }
+
+ private static void AddStat(RowDescriptor row, string label, string value)
+ {
+ row.RelativeItem().Padding(4).Border(1).BorderColor("#e2e8f0").Padding(8).Column(col =>
+ {
+ col.Item().Text(label).FontSize(8);
+ col.Item().Text(value).FontSize(16).Bold();
+ });
+ }
+}
+
+public sealed class AppendixSection : IPdfSectionComposer
+{
+ private static readonly (string Term, string Desc)[] Glossary =
+ [
+ ("Crawl", "URLs fetched by the site spider (status codes, titles, inlinks)."),
+ ("Lighthouse", "Lab Core Web Vitals audit (LCP, CLS, TBT, and category scores)."),
+ ("Google Search Console", "Queries, pages, clicks, impressions, and average position from GSC."),
+ ("Google Analytics 4", "Sessions, users, and engagement from GA4."),
+ ("Estimated", "Derived from crawl text only — not Google search volume or rankings."),
+ ("AI insights", "Optional LLM summaries — verify before client delivery."),
+ ];
+
+ public PdfSectionId SectionId => PdfSectionId.Appendix;
+ public string Title => "Appendix";
+
+ public bool IsAvailable(PdfRenderContext context) => context.Profile != PdfProfile.Executive;
+
+ public void Compose(IContainer container, PdfRenderContext context)
+ {
+ var model = context.Model;
+ container.Column(column =>
+ {
+ column.Item().Element(c => PdfTheme.SectionTitle(c, Title));
+ column.Item().Text($"Report ID: {model.ReportId}").FontSize(9);
+ column.Item().PaddingTop(8).Text("Data source glossary").FontSize(11).Bold();
+ foreach (var (term, desc) in Glossary)
+ {
+ column.Item().PaddingTop(4).Row(row =>
+ {
+ row.ConstantItem(120).Text(term).FontSize(9).Bold();
+ row.RelativeItem().Text(desc).FontSize(8);
+ });
+ }
+ if (model.LinkSamples.Count > 0)
+ {
+ column.Item().PaddingTop(12).Text("Crawled URLs (sample)").FontSize(11).Bold();
+ foreach (var link in model.LinkSamples)
+ {
+ column.Item().PaddingTop(3).Row(row =>
+ {
+ row.RelativeItem(3).Text(link.Url).FontSize(7).FontColor(PdfTheme.AccentColor);
+ row.ConstantItem(36).AlignRight().Text(link.Status).FontSize(7).Bold();
+ row.RelativeItem(2).Text(link.Title).FontSize(7).FontColor(PdfTheme.MutedColor);
+ });
+ }
+ }
+ });
+ }
+}
diff --git a/services/FileService/src/FileService.Rendering/Templates/CoverPageTemplate.cs b/services/FileService/src/FileService.Rendering/Templates/CoverPageTemplate.cs
new file mode 100644
index 00000000..3e18da2b
--- /dev/null
+++ b/services/FileService/src/FileService.Rendering/Templates/CoverPageTemplate.cs
@@ -0,0 +1,91 @@
+using FileService.Domain.Models;
+using FileService.Rendering.Charts;
+using QuestPDF.Fluent;
+using QuestPDF.Helpers;
+using QuestPDF.Infrastructure;
+
+namespace FileService.Rendering.Templates;
+
+public static class CoverPageTemplate
+{
+ public static void Compose(IContainer container, AuditReportModel model, PdfProfile profile)
+ {
+ container.Column(column =>
+ {
+ column.Item().Height(120).Background(PdfTheme.CoverBandColor).Padding(32).Column(band =>
+ {
+ if (model.Branding.Enabled)
+ {
+ band.Item().Row(row =>
+ {
+ if (model.Branding.LogoBytes is { Length: > 0 })
+ {
+ row.ConstantItem(80).Height(40).Image(model.Branding.LogoBytes).FitArea();
+ }
+ row.RelativeItem().AlignRight().Column(col =>
+ {
+ if (!string.IsNullOrWhiteSpace(model.Branding.AgencyName))
+ {
+ col.Item().AlignRight().Text(model.Branding.AgencyName).FontSize(11).FontColor(Colors.White);
+ }
+ if (!string.IsNullOrWhiteSpace(model.Branding.AgencySubtitle))
+ {
+ col.Item().AlignRight().Text(model.Branding.AgencySubtitle).FontSize(9).FontColor(Colors.Grey.Lighten2);
+ }
+ });
+ });
+ }
+ band.Item().PaddingTop(16).Text(model.ReportTitle).FontSize(14).FontColor(Colors.Grey.Lighten2);
+ });
+
+ column.Item().Padding(40).Column(body =>
+ {
+ body.Item().Text(model.SiteName).FontSize(PdfTheme.CoverTitleSize).Bold().FontColor(PdfTheme.TextColor);
+ if (!string.IsNullOrWhiteSpace(model.GeneratedAt))
+ {
+ body.Item().PaddingTop(8).Text($"Report generated {model.GeneratedAt}").FontSize(11).FontColor(PdfTheme.MutedColor);
+ }
+
+ if (model.HealthScore is not null)
+ {
+ body.Item().PaddingTop(24).Row(row =>
+ {
+ row.AutoItem().Text(model.HealthScore.ToString()!).FontSize(56).Bold()
+ .FontColor(PdfTheme.ScoreColor(model.HealthScore));
+ row.AutoItem().PaddingLeft(8).AlignBottom().PaddingBottom(8).Text("/ 100").FontSize(18).FontColor(PdfTheme.MutedColor);
+ row.RelativeItem().PaddingLeft(24).AlignMiddle().Column(col =>
+ {
+ col.Item().Text("Site health score").FontSize(12).FontColor(PdfTheme.MutedColor);
+ col.Item().Text(model.ScoreBand).FontSize(16).Bold().FontColor(PdfTheme.ScoreColor(model.HealthScore));
+ });
+ });
+ }
+
+ body.Item().PaddingTop(20).Row(row =>
+ {
+ foreach (var (priority, count) in model.IssueCounts.Where(kv => kv.Value > 0))
+ {
+ row.AutoItem().PaddingRight(8).Background(PdfTheme.PriorityColors.GetValueOrDefault(priority, PdfTheme.MutedColor))
+ .PaddingHorizontal(10).PaddingVertical(6).Text($"{priority.ToUpperInvariant()}: {count}")
+ .FontSize(9).Bold().FontColor(Colors.White);
+ }
+ });
+
+ if (model.TotalIssueCount > 0)
+ {
+ body.Item().PaddingTop(12).Text($"{model.TotalIssueCount} findings across {model.CategoryScores.Count} categories")
+ .FontSize(10).FontColor(PdfTheme.MutedColor);
+ }
+ });
+
+ column.Item().AlignBottom().Padding(32).Column(footer =>
+ {
+ footer.Item().Text("Confidential — prepared for client review.").FontSize(9).FontColor(PdfTheme.MutedColor);
+ if (model.Branding.Enabled && !string.IsNullOrWhiteSpace(model.Branding.AgencyName))
+ {
+ footer.Item().PaddingTop(4).Text($"Prepared by {model.Branding.AgencyName}").FontSize(9).Bold();
+ }
+ });
+ });
+ }
+}
diff --git a/services/FileService/tests/FileService.Tests/ApiErrorPathTests.cs b/services/FileService/tests/FileService.Tests/ApiErrorPathTests.cs
new file mode 100644
index 00000000..f3c4bb2d
--- /dev/null
+++ b/services/FileService/tests/FileService.Tests/ApiErrorPathTests.cs
@@ -0,0 +1,92 @@
+using System.Net;
+using System.Text.Json;
+using FileService.Application.Clients;
+using FileService.Domain.Models;
+using Microsoft.AspNetCore.Mvc.Testing;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
+
+namespace FileService.Tests;
+
+public class ApiErrorPathTests
+{
+ [Fact]
+ public async Task Pdf_returns_404_when_report_missing()
+ {
+ await using var factory = CreateFactory(new MissingPayloadClient());
+ var client = factory.CreateClient();
+
+ var response = await client.GetAsync("/v1/reports/999/pdf");
+
+ Assert.Equal(HttpStatusCode.NotFound, response.StatusCode);
+ }
+
+ [Fact]
+ public async Task Workbook_returns_404_when_report_missing()
+ {
+ await using var factory = CreateFactory(new MissingPayloadClient());
+ var client = factory.CreateClient();
+
+ var response = await client.GetAsync("/v1/reports/999/workbook");
+
+ Assert.Equal(HttpStatusCode.NotFound, response.StatusCode);
+ }
+
+ [Fact]
+ public async Task Pdf_returns_502_when_upstream_fails()
+ {
+ await using var factory = CreateFactory(new FailingPayloadClient());
+ var client = factory.CreateClient();
+
+ var response = await client.GetAsync("/v1/reports/1/pdf");
+
+ Assert.Equal(HttpStatusCode.BadGateway, response.StatusCode);
+ }
+
+ [Fact]
+ public async Task By_domain_pdf_returns_404_when_domain_unknown()
+ {
+ await using var factory = CreateFactory(new MissingPayloadClient());
+ var client = factory.CreateClient();
+
+ var response = await client.GetAsync("/v1/reports/by-domain/unknown.test/pdf");
+
+ Assert.Equal(HttpStatusCode.NotFound, response.StatusCode);
+ }
+
+ private static WebApplicationFactory CreateFactory(IReportDataClient reportClient) =>
+ new WebApplicationFactory().WithWebHostBuilder(builder =>
+ {
+ builder.ConfigureServices(services =>
+ {
+ services.RemoveAll();
+ services.RemoveAll();
+ services.AddSingleton(reportClient);
+ services.AddSingleton();
+ });
+ });
+
+ private sealed class MissingPayloadClient : IReportDataClient
+ {
+ public Task> ListReportsAsync(CancellationToken cancellationToken = default) =>
+ Task.FromResult>([]);
+
+ public Task GetPayloadAsync(int reportId, CancellationToken cancellationToken = default) =>
+ Task.FromResult(null);
+ }
+
+ private sealed class FailingPayloadClient : IReportDataClient
+ {
+ public Task> ListReportsAsync(CancellationToken cancellationToken = default) =>
+ throw new NotSupportedException();
+
+ public Task GetPayloadAsync(int reportId, CancellationToken cancellationToken = default) =>
+ throw new HttpRequestException("Report API returned 502: unavailable", null, HttpStatusCode.BadGateway);
+ }
+
+ private sealed class StubBrandingClient : IAppSettingsClient
+ {
+ public Task GetBrandingAsync(bool enabled, CancellationToken cancellationToken = default) =>
+ Task.FromResult(new PdfBrandingModel { Enabled = false });
+ }
+}
diff --git a/services/FileService/tests/FileService.Tests/ApiIntegrationTests.cs b/services/FileService/tests/FileService.Tests/ApiIntegrationTests.cs
new file mode 100644
index 00000000..1ae1185e
--- /dev/null
+++ b/services/FileService/tests/FileService.Tests/ApiIntegrationTests.cs
@@ -0,0 +1,133 @@
+using System.Net;
+using System.Text.Json;
+using FileService.Application.Clients;
+using FileService.Domain.Models;
+using Microsoft.AspNetCore.Mvc.Testing;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.DependencyInjection.Extensions;
+
+namespace FileService.Tests;
+
+public class ApiIntegrationTests : IClassFixture>
+{
+ private readonly WebApplicationFactory _factory;
+
+ public ApiIntegrationTests(WebApplicationFactory factory)
+ {
+ _factory = factory.WithWebHostBuilder(builder =>
+ {
+ builder.ConfigureServices(services =>
+ {
+ services.RemoveAll();
+ services.RemoveAll();
+ services.AddSingleton();
+ services.AddSingleton();
+ });
+ });
+ }
+
+ [Fact]
+ public async Task Health_returns_ok()
+ {
+ var client = _factory.CreateClient();
+ var response = await client.GetAsync("/health");
+ Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+ }
+
+ [Fact]
+ public async Task Swagger_json_available_in_development()
+ {
+ var client = _factory.CreateClient();
+ var response = await client.GetAsync("/swagger/v1/swagger.json");
+ Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+ var json = await response.Content.ReadAsStringAsync();
+ Assert.Contains("FileService API", json);
+ Assert.Contains("/v1/reports/{reportId}/pdf", json);
+ }
+
+ [Fact]
+ public async Task Report_pdf_returns_pdf_bytes()
+ {
+ var client = _factory.CreateClient();
+ var response = await client.GetAsync("/v1/reports/1/pdf?disposition=inline");
+ Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+ Assert.Equal("application/pdf", response.Content.Headers.ContentType?.MediaType);
+ var bytes = await response.Content.ReadAsByteArrayAsync();
+ Assert.True(bytes.Length > 4);
+ Assert.Equal("%PDF", System.Text.Encoding.ASCII.GetString(bytes, 0, 4));
+ }
+
+ [Fact]
+ public async Task Premium_profile_with_branding_returns_pdf()
+ {
+ var client = _factory.CreateClient();
+ var response = await client.GetAsync("/v1/reports/1/pdf?profile=premium&branding=true&disposition=inline");
+ Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+ }
+
+ [Fact]
+ public async Task By_domain_pdf_resolves_report()
+ {
+ var client = _factory.CreateClient();
+ var response = await client.GetAsync("/v1/reports/by-domain/example.com/pdf");
+ Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+ }
+
+ [Fact]
+ public async Task Report_workbook_returns_xlsx_bytes()
+ {
+ var client = _factory.CreateClient();
+ var response = await client.GetAsync("/v1/reports/1/workbook");
+ Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+ Assert.Equal(
+ "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+ response.Content.Headers.ContentType?.MediaType);
+ var bytes = await response.Content.ReadAsByteArrayAsync();
+ Assert.True(bytes.Length > 4);
+ Assert.Equal("PK", System.Text.Encoding.ASCII.GetString(bytes, 0, 2));
+ }
+
+ [Fact]
+ public async Task By_domain_workbook_resolves_report()
+ {
+ var client = _factory.CreateClient();
+ var response = await client.GetAsync("/v1/reports/by-domain/example.com/workbook");
+ Assert.Equal(HttpStatusCode.OK, response.StatusCode);
+ }
+
+ private sealed class FakeReportDataClient : IReportDataClient
+ {
+ public Task> ListReportsAsync(CancellationToken cancellationToken = default)
+ {
+ IReadOnlyList rows =
+ [
+ new ReportListRow { Id = 1, CanonicalDomain = "example.com", SiteName = "example.com" },
+ ];
+ return Task.FromResult(rows);
+ }
+
+ public async Task GetPayloadAsync(int reportId, CancellationToken cancellationToken = default)
+ {
+ var json = await File.ReadAllTextAsync(Path.Combine("fixtures", "full-payload.json"), cancellationToken);
+ using var doc = JsonDocument.Parse(json);
+ return doc.RootElement.Clone();
+ }
+ }
+
+ private sealed class FakeAppSettingsClient : IAppSettingsClient
+ {
+ public Task GetBrandingAsync(bool enabled, CancellationToken cancellationToken = default)
+ {
+ if (!enabled)
+ {
+ return Task.FromResult(new PdfBrandingModel { Enabled = false });
+ }
+ return Task.FromResult(new PdfBrandingModel
+ {
+ Enabled = true,
+ AgencyName = "Test Agency",
+ AgencySubtitle = "Audits",
+ });
+ }
+ }
+}
diff --git a/services/FileService/tests/FileService.Tests/AppSettingsClientTests.cs b/services/FileService/tests/FileService.Tests/AppSettingsClientTests.cs
new file mode 100644
index 00000000..45a8b0d7
--- /dev/null
+++ b/services/FileService/tests/FileService.Tests/AppSettingsClientTests.cs
@@ -0,0 +1,124 @@
+using System.Net;
+using FileService.Application.Clients;
+using FileService.Application.Options;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+
+namespace FileService.Tests;
+
+public class AppSettingsClientTests
+{
+ [Fact]
+ public async Task GetBrandingAsync_disabled_returns_empty_model()
+ {
+ var client = new AppSettingsClient(
+ TestHttpHandler.CreateClient(_ => throw new InvalidOperationException("should not call")),
+ Options.Create(new ReportApiOptions { BaseUrl = "http://report-api.test" }),
+ new FakeLogoFetcher(),
+ NullLogger.Instance);
+
+ var brand = await client.GetBrandingAsync(false);
+
+ Assert.False(brand.Enabled);
+ }
+
+ [Fact]
+ public async Task GetBrandingAsync_loads_brand_keys_and_logo()
+ {
+ using var http = TestHttpHandler.CreateClient(req =>
+ {
+ var path = req.RequestUri!.AbsolutePath + req.RequestUri.Query;
+ return path switch
+ {
+ var p when p.Contains("brand_name") => TestHttpHandler.Json("""{"value":"Agency Co"}"""),
+ var p when p.Contains("brand_subtitle") => TestHttpHandler.Json("""{"value":"Audits"}"""),
+ var p when p.Contains("brand_logo_url") => TestHttpHandler.Json("""{"value":"https://cdn/logo.png"}"""),
+ _ => new HttpResponseMessage(HttpStatusCode.NotFound),
+ };
+ });
+ var logoFetcher = new FakeLogoFetcher { Bytes = [1, 2, 3] };
+ var client = new AppSettingsClient(
+ http,
+ Options.Create(new ReportApiOptions { BaseUrl = "http://report-api.test" }),
+ logoFetcher,
+ NullLogger.Instance);
+
+ var brand = await client.GetBrandingAsync(true);
+
+ Assert.True(brand.Enabled);
+ Assert.Equal("Agency Co", brand.AgencyName);
+ Assert.Equal("Audits", brand.AgencySubtitle);
+ Assert.Equal([1, 2, 3], brand.LogoBytes);
+ Assert.Equal("https://cdn/logo.png", logoFetcher.LastUrl);
+ }
+
+ [Fact]
+ public async Task GetBrandingAsync_ignores_failed_setting_requests()
+ {
+ using var http = TestHttpHandler.CreateClient(_ => new HttpResponseMessage(HttpStatusCode.InternalServerError));
+ var client = new AppSettingsClient(
+ http,
+ Options.Create(new ReportApiOptions { BaseUrl = "http://report-api.test" }),
+ new FakeLogoFetcher(),
+ NullLogger.Instance);
+
+ var brand = await client.GetBrandingAsync(true);
+
+ Assert.True(brand.Enabled);
+ Assert.Equal("", brand.AgencyName);
+ }
+
+ [Fact]
+ public async Task LogoFetcher_returns_null_for_empty_url()
+ {
+ var fetcher = new LogoFetcher(
+ TestHttpHandler.CreateClient(_ => throw new InvalidOperationException()),
+ NullLogger.Instance);
+
+ var bytes = await fetcher.FetchAsync(" ");
+
+ Assert.Null(bytes);
+ }
+
+ [Fact]
+ public async Task LogoFetcher_returns_bytes_when_small_enough()
+ {
+ var fetcher = new LogoFetcher(
+ TestHttpHandler.CreateClient(_ => new HttpResponseMessage(HttpStatusCode.OK)
+ {
+ Content = new ByteArrayContent([5, 6, 7]),
+ }),
+ NullLogger.Instance);
+
+ var bytes = await fetcher.FetchAsync("https://cdn/logo.png");
+
+ Assert.Equal([5, 6, 7], bytes);
+ }
+
+ [Fact]
+ public async Task LogoFetcher_returns_null_when_response_too_large()
+ {
+ var fetcher = new LogoFetcher(
+ TestHttpHandler.CreateClient(_ => new HttpResponseMessage(HttpStatusCode.OK)
+ {
+ Content = new ByteArrayContent(new byte[512 * 1024 + 1]),
+ }),
+ NullLogger.Instance);
+
+ var bytes = await fetcher.FetchAsync("https://cdn/huge.png");
+
+ Assert.Null(bytes);
+ }
+
+ private sealed class FakeLogoFetcher : ILogoFetcher
+ {
+ public byte[]? Bytes { get; set; }
+ public string? LastUrl { get; private set; }
+
+ public Task FetchAsync(string? url, CancellationToken cancellationToken = default)
+ {
+ LastUrl = url;
+ return Task.FromResult(Bytes);
+ }
+ }
+}
diff --git a/services/FileService/tests/FileService.Tests/AuditPdfGeneratorTests.cs b/services/FileService/tests/FileService.Tests/AuditPdfGeneratorTests.cs
new file mode 100644
index 00000000..9b750de2
--- /dev/null
+++ b/services/FileService/tests/FileService.Tests/AuditPdfGeneratorTests.cs
@@ -0,0 +1,50 @@
+using System.Text.Json;
+using FileService.Application.Mapping;
+using FileService.Domain.Models;
+using FileService.Rendering;
+using FileService.Rendering.Composition;
+
+namespace FileService.Tests;
+
+public class AuditPdfGeneratorTests
+{
+ private static readonly PdfBrandingModel NoBranding = new() { Enabled = false };
+
+ [Theory]
+ [InlineData(PdfProfile.Executive)]
+ [InlineData(PdfProfile.Standard)]
+ [InlineData(PdfProfile.Full)]
+ [InlineData(PdfProfile.Premium)]
+ public void Generate_each_profile_produces_valid_pdf(PdfProfile profile)
+ {
+ var fixture = profile is PdfProfile.Full or PdfProfile.Premium
+ ? "full-payload.json"
+ : "minimal-payload.json";
+ var json = File.ReadAllText(Path.Combine("fixtures", fixture));
+ using var doc = JsonDocument.Parse(json);
+ var model = AuditReportMapper.Map(doc.RootElement, 1, profile, NoBranding).WithTableOfContents(profile);
+
+ var generator = new AuditPdfGenerator();
+ var bytes = generator.Generate(model, profile);
+
+ Assert.NotEmpty(bytes);
+ Assert.Equal("%PDF", System.Text.Encoding.ASCII.GetString(bytes, 0, 4));
+ }
+
+ [Fact]
+ public void Generate_with_branding_still_produces_pdf()
+ {
+ var json = File.ReadAllText(Path.Combine("fixtures", "minimal-payload.json"));
+ using var doc = JsonDocument.Parse(json);
+ var branding = new PdfBrandingModel
+ {
+ Enabled = true,
+ AgencyName = "Test Agency",
+ AgencySubtitle = "SEO Audits",
+ };
+ var model = AuditReportMapper.Map(doc.RootElement, 1, PdfProfile.Premium, branding).WithTableOfContents(PdfProfile.Premium);
+
+ var bytes = new AuditPdfGenerator().Generate(model, PdfProfile.Premium);
+ Assert.True(bytes.Length > 1000);
+ }
+}
diff --git a/services/FileService/tests/FileService.Tests/AuditReportMapperTests.cs b/services/FileService/tests/FileService.Tests/AuditReportMapperTests.cs
new file mode 100644
index 00000000..6e2e6d92
--- /dev/null
+++ b/services/FileService/tests/FileService.Tests/AuditReportMapperTests.cs
@@ -0,0 +1,113 @@
+using System.Text.Json;
+using FileService.Application.Domain;
+using FileService.Application.Mapping;
+using FileService.Domain.Models;
+using FileService.Rendering.Composition;
+
+namespace FileService.Tests;
+
+public class AuditReportMapperTests
+{
+ private static readonly PdfBrandingModel NoBranding = new() { Enabled = false };
+
+ [Fact]
+ public void Map_minimal_payload_populates_core_fields()
+ {
+ var json = File.ReadAllText(Path.Combine("fixtures", "minimal-payload.json"));
+ using var doc = JsonDocument.Parse(json);
+ var model = AuditReportMapper.Map(doc.RootElement, 42, PdfProfile.Standard, NoBranding);
+
+ Assert.Equal(42, model.ReportId);
+ Assert.Equal("example.com", model.SiteName);
+ Assert.Equal(72, model.HealthScore);
+ Assert.Equal(3, model.Issues.Count);
+ Assert.Contains(model.Issues, i => i.Priority.Equals("critical", StringComparison.OrdinalIgnoreCase));
+ Assert.NotEmpty(model.ExecutiveSummary.Summary);
+ Assert.Equal(2, model.CategoryScores.Count);
+ Assert.NotEmpty(model.Issues[0].Headline);
+ }
+
+ [Fact]
+ public void Map_executive_profile_limits_issues()
+ {
+ var json = File.ReadAllText(Path.Combine("fixtures", "minimal-payload.json"));
+ using var doc = JsonDocument.Parse(json);
+ var model = AuditReportMapper.Map(doc.RootElement, 1, PdfProfile.Executive, NoBranding);
+
+ Assert.True(model.Issues.Count <= 6);
+ }
+
+ [Fact]
+ public void Map_full_payload_includes_analytics_chapters()
+ {
+ var json = File.ReadAllText(Path.Combine("fixtures", "full-payload.json"));
+ using var doc = JsonDocument.Parse(json);
+ var model = AuditReportMapper.Map(doc.RootElement, 2, PdfProfile.Premium, NoBranding);
+
+ Assert.NotNull(model.Lighthouse);
+ Assert.NotNull(model.SearchVisibility);
+ Assert.NotNull(model.Traffic);
+ Assert.NotNull(model.Security);
+ Assert.NotNull(model.Content);
+ Assert.NotNull(model.Indexation);
+ Assert.Equal(2, model.LinkSamples.Count);
+ }
+}
+
+public class DomainResolverTests
+{
+ [Fact]
+ public void Matches_canonical_domain()
+ {
+ var rows = new List
+ {
+ new() { Id = 5, CanonicalDomain = "example.com", SiteName = "Example" },
+ };
+ Assert.Equal(5, DomainResolver.ResolveReportId(rows, "example.com"));
+ }
+
+ [Fact]
+ public void Matches_slugified_site_name()
+ {
+ var rows = new List
+ {
+ new() { Id = 7, CanonicalDomain = null, SiteName = "My Cool Site" },
+ };
+ Assert.Equal(7, DomainResolver.ResolveReportId(rows, "my-cool-site"));
+ }
+
+ [Fact]
+ public void Returns_null_for_empty_or_unknown_query()
+ {
+ var rows = new List { new() { Id = 1, CanonicalDomain = "ex.com" } };
+ Assert.Null(DomainResolver.ResolveReportId(rows, ""));
+ Assert.Null(DomainResolver.ResolveReportId(rows, "unknown.test"));
+ }
+
+ [Fact]
+ public void NormalizeDomainQuery_strips_trailing_punctuation()
+ {
+ Assert.Equal("example.com", DomainResolver.NormalizeDomainQuery("Example.COM,"));
+ }
+
+ [Fact]
+ public void Matches_slugified_canonical_host()
+ {
+ var row = new ReportListRow { Id = 2, CanonicalDomain = "www.example.com", SiteName = "Example" };
+ Assert.True(DomainResolver.DomainQueryMatchesRow(row, "www-example-com"));
+ }
+}
+
+public class TocBuilderTests
+{
+ [Fact]
+ public void Build_standard_profile_includes_findings()
+ {
+ var json = File.ReadAllText(Path.Combine("fixtures", "minimal-payload.json"));
+ using var doc = JsonDocument.Parse(json);
+ var model = AuditReportMapper.Map(doc.RootElement, 1, PdfProfile.Standard, new PdfBrandingModel { Enabled = false })
+ .WithTableOfContents(PdfProfile.Standard);
+
+ Assert.Contains(model.TableOfContents, e => e.SectionId == PdfSectionId.Findings);
+ }
+}
diff --git a/services/FileService/tests/FileService.Tests/AuditWorkbookGeneratorTests.cs b/services/FileService/tests/FileService.Tests/AuditWorkbookGeneratorTests.cs
new file mode 100644
index 00000000..3c50318d
--- /dev/null
+++ b/services/FileService/tests/FileService.Tests/AuditWorkbookGeneratorTests.cs
@@ -0,0 +1,63 @@
+using System.Text.Json;
+using ClosedXML.Excel;
+using FileService.Rendering;
+
+namespace FileService.Tests;
+
+public class AuditWorkbookGeneratorTests
+{
+ private static JsonElement LoadFixture(string name)
+ {
+ var json = File.ReadAllText(Path.Combine("fixtures", name));
+ using var doc = JsonDocument.Parse(json);
+ return doc.RootElement.Clone();
+ }
+
+ [Fact]
+ public void Generate_contains_expected_sheets()
+ {
+ var generator = new AuditWorkbookGenerator();
+ var bytes = generator.Generate(LoadFixture("full-payload.json"));
+
+ using var stream = new MemoryStream(bytes);
+ using var workbook = new XLWorkbook(stream);
+ var names = workbook.Worksheets.Select(w => w.Name).ToHashSet(StringComparer.Ordinal);
+
+ Assert.Contains("Internal URLs", names);
+ Assert.Contains("Links", names);
+ Assert.Contains("Issues", names);
+ Assert.Contains("Redirects", names);
+ Assert.Contains("Custom Fields", names);
+ }
+
+ [Fact]
+ public void Generate_custom_fields_sheet_has_dynamic_columns()
+ {
+ var generator = new AuditWorkbookGenerator();
+ var bytes = generator.Generate(LoadFixture("full-payload.json"));
+
+ using var stream = new MemoryStream(bytes);
+ using var workbook = new XLWorkbook(stream);
+ var sheet = workbook.Worksheet("Custom Fields");
+ var headerRow = sheet.Row(1).Cells().Select(c => c.GetString()).ToList();
+
+ Assert.Contains("price", headerRow);
+ Assert.Contains("sku", headerRow);
+ Assert.Contains("9.99", sheet.CellsUsed().Select(c => c.GetString()).Where(s => s is not null));
+ }
+
+ [Fact]
+ public void Generate_minimal_payload_produces_internal_urls_only()
+ {
+ var generator = new AuditWorkbookGenerator();
+ var bytes = generator.Generate(LoadFixture("minimal-payload.json"));
+
+ using var stream = new MemoryStream(bytes);
+ using var workbook = new XLWorkbook(stream);
+
+ var names = workbook.Worksheets.Select(w => w.Name).ToList();
+ Assert.Equal(2, workbook.Worksheets.Count);
+ Assert.Contains("Internal URLs", names);
+ Assert.Contains("Issues", names);
+ }
+}
diff --git a/services/FileService/tests/FileService.Tests/FileService.Tests.csproj b/services/FileService/tests/FileService.Tests/FileService.Tests.csproj
new file mode 100644
index 00000000..09822161
--- /dev/null
+++ b/services/FileService/tests/FileService.Tests/FileService.Tests.csproj
@@ -0,0 +1,35 @@
+
+
+
+ net10.0
+ enable
+ enable
+ false
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ PreserveNewest
+
+
+
+
\ No newline at end of file
diff --git a/services/FileService/tests/FileService.Tests/ReportDataClientTests.cs b/services/FileService/tests/FileService.Tests/ReportDataClientTests.cs
new file mode 100644
index 00000000..cfb5783f
--- /dev/null
+++ b/services/FileService/tests/FileService.Tests/ReportDataClientTests.cs
@@ -0,0 +1,88 @@
+using System.Net;
+using System.Text.Json;
+using FileService.Application.Clients;
+using FileService.Application.Options;
+using Microsoft.Extensions.Logging.Abstractions;
+using Microsoft.Extensions.Options;
+
+namespace FileService.Tests;
+
+public class ReportDataClientTests
+{
+ private static ReportDataClient CreateClient(HttpClient http) =>
+ new(http, Options.Create(new ReportApiOptions { BaseUrl = "http://report-api.test", TimeoutSeconds = 30 }),
+ NullLogger.Instance);
+
+ [Fact]
+ public async Task ListReportsAsync_parses_meta_response()
+ {
+ using var http = TestHttpHandler.CreateClient(_ =>
+ TestHttpHandler.Json("""{"reports":[{"id":3,"canonical_domain":"ex.com","site_name":"Ex","generated_at":"2025-01-01"}]}"""));
+ var client = CreateClient(http);
+
+ var rows = await client.ListReportsAsync();
+
+ Assert.Single(rows);
+ Assert.Equal(3, rows[0].Id);
+ Assert.Equal("ex.com", rows[0].CanonicalDomain);
+ }
+
+ [Fact]
+ public async Task ListReportsAsync_returns_empty_when_reports_missing()
+ {
+ using var http = TestHttpHandler.CreateClient(_ => TestHttpHandler.Json("{}"));
+ var client = CreateClient(http);
+
+ var rows = await client.ListReportsAsync();
+
+ Assert.Empty(rows);
+ }
+
+ [Fact]
+ public async Task GetPayloadAsync_returns_payload_element()
+ {
+ using var http = TestHttpHandler.CreateClient(_ =>
+ TestHttpHandler.Json("""{"payload":{"site_name":"Acme"}}"""));
+ var client = CreateClient(http);
+
+ var payload = await client.GetPayloadAsync(9);
+
+ Assert.NotNull(payload);
+ Assert.Equal("Acme", payload.Value.GetProperty("site_name").GetString());
+ }
+
+ [Fact]
+ public async Task GetPayloadAsync_returns_null_on_404()
+ {
+ using var http = TestHttpHandler.CreateClient(_ => new HttpResponseMessage(HttpStatusCode.NotFound));
+ var client = CreateClient(http);
+
+ var payload = await client.GetPayloadAsync(404);
+
+ Assert.Null(payload);
+ }
+
+ [Fact]
+ public async Task GetPayloadAsync_returns_null_when_payload_property_missing()
+ {
+ using var http = TestHttpHandler.CreateClient(_ => TestHttpHandler.Json("{}"));
+ var client = CreateClient(http);
+
+ var payload = await client.GetPayloadAsync(1);
+
+ Assert.Null(payload);
+ }
+
+ [Fact]
+ public async Task ListReportsAsync_throws_on_upstream_error()
+ {
+ using var http = TestHttpHandler.CreateClient(_ => new HttpResponseMessage(HttpStatusCode.BadGateway)
+ {
+ Content = new StringContent("upstream down"),
+ });
+ var client = CreateClient(http);
+
+ var ex = await Assert.ThrowsAsync(() => client.ListReportsAsync());
+ Assert.Contains("502", ex.Message);
+ }
+}
diff --git a/services/FileService/tests/FileService.Tests/ReportServiceTests.cs b/services/FileService/tests/FileService.Tests/ReportServiceTests.cs
new file mode 100644
index 00000000..27340c81
--- /dev/null
+++ b/services/FileService/tests/FileService.Tests/ReportServiceTests.cs
@@ -0,0 +1,95 @@
+using System.Text.Json;
+using FileService.Application.Clients;
+using FileService.Application.Services;
+using FileService.Domain.Models;
+using FileService.Rendering;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace FileService.Tests;
+
+public class ReportServiceTests
+{
+ [Fact]
+ public async Task PdfReportService_throws_when_payload_missing()
+ {
+ var service = new PdfReportService(
+ new NullPayloadClient(),
+ new StubBrandingClient(),
+ new AuditPdfGenerator(),
+ NullLogger.Instance);
+
+ await Assert.ThrowsAsync(() =>
+ service.GenerateByReportIdAsync(99, PdfProfile.Standard, branding: false));
+ }
+
+ [Fact]
+ public async Task PdfReportService_throws_when_domain_unresolved()
+ {
+ var service = new PdfReportService(
+ new EmptyReportsClient(),
+ new StubBrandingClient(),
+ new AuditPdfGenerator(),
+ NullLogger.Instance);
+
+ await Assert.ThrowsAsync(() =>
+ service.GenerateByDomainAsync("missing.test", PdfProfile.Standard, branding: false));
+ }
+
+ [Fact]
+ public async Task WorkbookReportService_throws_when_payload_missing()
+ {
+ var service = new WorkbookReportService(
+ new NullPayloadClient(),
+ new AuditWorkbookGenerator(),
+ NullLogger.Instance);
+
+ await Assert.ThrowsAsync(() => service.GenerateByReportIdAsync(99));
+ }
+
+ [Fact]
+ public async Task WorkbookReportService_generates_bytes_when_payload_exists()
+ {
+ using var doc = JsonDocument.Parse("""{"links":[{"url":"https://ex.com","status":"200"}]}""");
+ var service = new WorkbookReportService(
+ new FixedPayloadClient(doc.RootElement.Clone()),
+ new AuditWorkbookGenerator(),
+ NullLogger.Instance);
+
+ var bytes = await service.GenerateByReportIdAsync(1);
+
+ Assert.StartsWith("PK", System.Text.Encoding.ASCII.GetString(bytes, 0, 2));
+ }
+
+ private sealed class NullPayloadClient : IReportDataClient
+ {
+ public Task> ListReportsAsync(CancellationToken cancellationToken = default) =>
+ Task.FromResult>([]);
+
+ public Task GetPayloadAsync(int reportId, CancellationToken cancellationToken = default) =>
+ Task.FromResult(null);
+ }
+
+ private sealed class EmptyReportsClient : IReportDataClient
+ {
+ public Task> ListReportsAsync(CancellationToken cancellationToken = default) =>
+ Task.FromResult>([]);
+
+ public Task GetPayloadAsync(int reportId, CancellationToken cancellationToken = default) =>
+ throw new NotSupportedException();
+ }
+
+ private sealed class FixedPayloadClient(JsonElement payload) : IReportDataClient
+ {
+ public Task> ListReportsAsync(CancellationToken cancellationToken = default) =>
+ Task.FromResult>([]);
+
+ public Task GetPayloadAsync(int reportId, CancellationToken cancellationToken = default) =>
+ Task.FromResult(payload);
+ }
+
+ private sealed class StubBrandingClient : IAppSettingsClient
+ {
+ public Task GetBrandingAsync(bool enabled, CancellationToken cancellationToken = default) =>
+ Task.FromResult(new PdfBrandingModel { Enabled = enabled });
+ }
+}
diff --git a/services/FileService/tests/FileService.Tests/TestHttpHandler.cs b/services/FileService/tests/FileService.Tests/TestHttpHandler.cs
new file mode 100644
index 00000000..b07adca4
--- /dev/null
+++ b/services/FileService/tests/FileService.Tests/TestHttpHandler.cs
@@ -0,0 +1,25 @@
+using System.Net;
+
+namespace FileService.Tests;
+
+internal sealed class TestHttpHandler(Func responder) : HttpMessageHandler
+{
+ public IList Requests { get; } = [];
+
+ protected override Task SendAsync(
+ HttpRequestMessage request,
+ CancellationToken cancellationToken)
+ {
+ Requests.Add(request);
+ return Task.FromResult(responder(request));
+ }
+
+ public static HttpClient CreateClient(Func responder) =>
+ new(new TestHttpHandler(responder)) { BaseAddress = new Uri("http://report-api.test/") };
+
+ public static HttpResponseMessage Json(string body, HttpStatusCode status = HttpStatusCode.OK) =>
+ new(status)
+ {
+ Content = new StringContent(body, System.Text.Encoding.UTF8, "application/json"),
+ };
+}
diff --git a/services/FileService/tests/FileService.Tests/fixtures/full-payload.json b/services/FileService/tests/FileService.Tests/fixtures/full-payload.json
new file mode 100644
index 00000000..9ac6728f
--- /dev/null
+++ b/services/FileService/tests/FileService.Tests/fixtures/full-payload.json
@@ -0,0 +1,128 @@
+{
+ "site_name": "example.com",
+ "report_title": "Technical SEO Audit Report",
+ "report_generated_at": "2025-06-01T12:00:00Z",
+ "overall_score": 72,
+ "summary": {
+ "total_urls": 200,
+ "indexable": 180,
+ "total_issues": 45,
+ "critical_issues": 2
+ },
+ "status_counts": {
+ "200": 180,
+ "301": 10,
+ "404": 5
+ },
+ "executive_summary": {
+ "summary": "The site has several high-priority SEO issues.",
+ "source": "deterministic",
+ "priorities": ["Fix broken links", "Improve meta descriptions"]
+ },
+ "report_meta": {
+ "data_sources": ["Crawl", "Google Search Console", "Google Analytics 4"],
+ "google_fetched_at": "2025-06-01T11:00:00Z",
+ "crawl_scope": {
+ "pages_crawled": 150,
+ "max_pages_configured": 500,
+ "render_mode": "auto"
+ }
+ },
+ "categories": [
+ {
+ "name": "technical_seo",
+ "score": 68,
+ "issues": [
+ {
+ "priority": "critical",
+ "message": "Server error on https://example.com/",
+ "url": "https://example.com/",
+ "recommendation": "Fix 500 response.",
+ "gsc_clicks": 120,
+ "gsc_impressions": 4500
+ }
+ ]
+ }
+ ],
+ "lighthouse_summary": {
+ "url": "https://example.com/",
+ "performance": 85,
+ "accessibility": 92,
+ "best_practices": 88,
+ "seo": 90
+ },
+ "lighthouse_human_summary": "Core Web Vitals are generally good.",
+ "lighthouse_diagnostics": [
+ { "title": "Largest Contentful Paint", "description": "2.1s" }
+ ],
+ "search_performance": {
+ "queries": [
+ { "query": "example brand", "clicks": "50", "impressions": "1200" }
+ ],
+ "pages": [
+ { "page": "https://example.com/", "clicks": "200", "impressions": "8000" }
+ ]
+ },
+ "ga4": {
+ "channels": [
+ { "channel": "Organic Search", "sessions": "1200" }
+ ],
+ "devices": [
+ { "device": "mobile", "sessions": "800" }
+ ]
+ },
+ "security_findings": [
+ {
+ "severity": "high",
+ "finding_type": "missing_header",
+ "url": "https://example.com/",
+ "message": "Missing Content-Security-Policy header"
+ }
+ ],
+ "content_analytics": {
+ "word_count_stats": { "mean": 450, "median": 380 },
+ "thin_content_count": 12,
+ "top_keywords_site": [
+ { "word": "services", "count": "45" }
+ ]
+ },
+ "indexation_coverage": {
+ "indexable": 180,
+ "non_indexable": 15,
+ "blocked": 5
+ },
+ "links": [
+ { "url": "https://example.com/", "status": "200", "title": "Home", "inlinks": 1, "outlinks": 2 },
+ {
+ "url": "https://example.com/products",
+ "status": "200",
+ "title": "Products",
+ "custom_extract": "SKU-1",
+ "custom_fields": "{\"price\":\"9.99\",\"sku\":\"SKU-1\"}"
+ }
+ ],
+ "link_edges": [
+ {
+ "from_url": "https://example.com/",
+ "to_url": "https://example.com/about",
+ "anchor_text": "About",
+ "rel": "",
+ "is_nofollow": false,
+ "is_sponsored": false,
+ "is_ugc": false,
+ "link_type": "internal",
+ "position": 1
+ }
+ ],
+ "redirects": [
+ {
+ "url": "https://example.com/old",
+ "message": "301 redirect chain",
+ "priority": "medium",
+ "recommendation": "Update internal links"
+ }
+ ],
+ "top_pages": [
+ { "url": "https://example.com/" }
+ ]
+}
diff --git a/services/FileService/tests/FileService.Tests/fixtures/minimal-payload.json b/services/FileService/tests/FileService.Tests/fixtures/minimal-payload.json
new file mode 100644
index 00000000..36afa384
--- /dev/null
+++ b/services/FileService/tests/FileService.Tests/fixtures/minimal-payload.json
@@ -0,0 +1,73 @@
+{
+ "site_name": "example.com",
+ "report_title": "Technical SEO Audit Report",
+ "report_generated_at": "2025-06-01T12:00:00Z",
+ "overall_score": 72,
+ "executive_summary": {
+ "summary": "The site has several high-priority SEO issues.",
+ "source": "deterministic",
+ "priorities": ["Fix broken links", "Improve meta descriptions"],
+ "top_issues": [
+ {
+ "category": "Technical SEO",
+ "priority": "high",
+ "message": "Missing title tag",
+ "url": "https://example.com/page",
+ "recommendation": "Add a unique title."
+ }
+ ]
+ },
+ "report_meta": {
+ "data_sources": ["Crawl", "Google Search Console"],
+ "crawl_scope": {
+ "pages_crawled": 150,
+ "max_pages_configured": 500
+ }
+ },
+ "categories": [
+ {
+ "name": "technical_seo",
+ "score": 68,
+ "issues": [
+ {
+ "priority": "critical",
+ "message": "Server error on homepage",
+ "url": "https://example.com/",
+ "recommendation": "Fix 500 response."
+ },
+ {
+ "priority": "high",
+ "message": "Missing meta description",
+ "url": "https://example.com/about",
+ "recommendation": "Add meta description."
+ }
+ ]
+ },
+ {
+ "name": "content",
+ "score": 80,
+ "issues": [
+ {
+ "priority": "medium",
+ "message": "Thin content",
+ "url": "https://example.com/blog/post",
+ "recommendation": "Expand content."
+ }
+ ]
+ }
+ ],
+ "lighthouse_summary": {
+ "url": "https://example.com/",
+ "performance": 85,
+ "accessibility": 92,
+ "best_practices": 88,
+ "seo": 90
+ },
+ "top_pages": [
+ { "url": "https://example.com/" },
+ { "url": "https://example.com/about" }
+ ],
+ "links": [
+ { "url": "https://example.com/", "status": "200", "title": "Home" }
+ ]
+}
diff --git a/src/website_profiling/api/__init__.py b/src/website_profiling/api/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/website_profiling/api/deps.py b/src/website_profiling/api/deps.py
new file mode 100644
index 00000000..8675c831
--- /dev/null
+++ b/src/website_profiling/api/deps.py
@@ -0,0 +1,19 @@
+"""Shared FastAPI dependencies."""
+from __future__ import annotations
+
+from typing import Iterator
+
+from psycopg import Connection
+
+from website_profiling.db.pool import db_session
+
+
+def get_db() -> Iterator[Connection]:
+ """Yield a synchronous psycopg connection from the pool.
+
+ Declare route handlers as plain ``def`` (not ``async def``) so FastAPI
+ runs them in a thread pool automatically — this matches the existing
+ synchronous codebase and requires no pool migration.
+ """
+ with db_session() as conn:
+ yield conn
diff --git a/src/website_profiling/api/main.py b/src/website_profiling/api/main.py
new file mode 100644
index 00000000..e26b2d4d
--- /dev/null
+++ b/src/website_profiling/api/main.py
@@ -0,0 +1,117 @@
+"""FastAPI application entry point."""
+from __future__ import annotations
+
+import os
+from contextlib import asynccontextmanager
+from typing import AsyncIterator
+
+from fastapi import FastAPI
+from fastapi.middleware.cors import CORSMiddleware
+
+from .routers import (
+ alerts,
+ chat,
+ compare,
+ config,
+ content,
+ crawl,
+ dashboards,
+ filters,
+ health,
+ integrations,
+ issues,
+ keywords,
+ logs,
+ mcp_tools,
+ ollama,
+ page_coach,
+ page_markdown,
+ pipeline,
+ portfolio,
+ properties,
+ report,
+ report_audit_tool,
+ report_export,
+ report_portfolio,
+ schedule,
+)
+
+
+@asynccontextmanager
+async def _lifespan(app: FastAPI) -> AsyncIterator[None]:
+ yield
+ # Close the psycopg connection pool on shutdown.
+ try:
+ from website_profiling.db.pool import close_db_pool
+
+ close_db_pool()
+ except Exception as exc:
+ import logging
+ logging.getLogger(__name__).warning("Error closing DB pool on shutdown: %s", exc)
+
+
+app = FastAPI(
+ title="Website Profiling API",
+ version="1.0.0",
+ lifespan=_lifespan,
+)
+
+# CORS — only added when FASTAPI_ALLOWED_ORIGINS is set (local Swagger in dev).
+_origins_raw = os.getenv("FASTAPI_ALLOWED_ORIGINS", "").strip()
+if _origins_raw:
+ _origins = [o.strip() for o in _origins_raw.split(",") if o.strip()]
+ if "*" in _origins:
+ raise RuntimeError(
+ "FASTAPI_ALLOWED_ORIGINS cannot contain '*' when allow_credentials=True. "
+ "List explicit origins (e.g. http://localhost:3000) instead."
+ )
+ app.add_middleware(
+ CORSMiddleware,
+ allow_origins=_origins,
+ allow_credentials=True,
+ allow_methods=["*"],
+ allow_headers=["*"],
+ )
+
+# ── Core routes ───────────────────────────────────────────────────────────────
+app.include_router(health.router, prefix="/api")
+app.include_router(report.router, prefix="/api")
+
+# ── Batch B: Pipeline jobs ────────────────────────────────────────────────────
+app.include_router(pipeline.router, prefix="/api")
+
+# ── Batch C: Chat (SSE + sessions) ───────────────────────────────────────────
+app.include_router(chat.router, prefix="/api")
+
+# ── Batch D: Crawl ───────────────────────────────────────────────────────────
+app.include_router(crawl.router, prefix="/api")
+
+# ── Batch E: Config (pipeline, LLM, secrets, app-settings) ───────────────────
+app.include_router(config.router, prefix="/api")
+
+# ── Batch F: Properties ──────────────────────────────────────────────────────
+app.include_router(properties.router, prefix="/api")
+
+# ── Batch G: Dashboards + Filters ────────────────────────────────────────────
+app.include_router(dashboards.router, prefix="/api")
+app.include_router(filters.router, prefix="/api")
+
+# ── Batch H: Google + Bing integrations ──────────────────────────────────────
+app.include_router(integrations.router, prefix="/api")
+
+# ── Batch I: Issues, keywords, content, page markdown, long-tail ─────────────
+app.include_router(issues.router, prefix="/api")
+app.include_router(keywords.router, prefix="/api")
+app.include_router(content.router, prefix="/api")
+app.include_router(page_markdown.router, prefix="/api")
+app.include_router(ollama.router, prefix="/api")
+app.include_router(mcp_tools.router, prefix="/api")
+app.include_router(portfolio.router, prefix="/api")
+app.include_router(alerts.router, prefix="/api")
+app.include_router(schedule.router, prefix="/api")
+app.include_router(logs.router, prefix="/api")
+app.include_router(compare.router, prefix="/api")
+app.include_router(page_coach.router, prefix="/api")
+app.include_router(report_audit_tool.router, prefix="/api")
+app.include_router(report_export.router, prefix="/api")
+app.include_router(report_portfolio.router, prefix="/api")
diff --git a/src/website_profiling/api/routers/__init__.py b/src/website_profiling/api/routers/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/website_profiling/api/routers/alerts.py b/src/website_profiling/api/routers/alerts.py
new file mode 100644
index 00000000..ae95963f
--- /dev/null
+++ b/src/website_profiling/api/routers/alerts.py
@@ -0,0 +1,32 @@
+"""Property alert checks — /api/alerts/*."""
+from __future__ import annotations
+
+from typing import Annotated, Any
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from psycopg import Connection
+
+from ..deps import get_db
+
+router = APIRouter(tags=["alerts"])
+
+DbDep = Annotated[Connection, Depends(get_db)]
+
+
+@router.post("/alerts/check")
+def alerts_check(
+ conn: DbDep,
+ propertyId: int = Query(...),
+) -> dict[str, Any]:
+ if not propertyId:
+ raise HTTPException(status_code=400, detail="propertyId required")
+ try:
+ from website_profiling.tools.alerts_runner import run_alerts_for_property
+
+ return run_alerts_for_property(conn, propertyId)
+ except ImportError:
+ import logging
+ logging.getLogger(__name__).debug("alerts_runner module not available in this build")
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
+ return {"ok": True, "checked": 0}
diff --git a/src/website_profiling/api/routers/chat.py b/src/website_profiling/api/routers/chat.py
new file mode 100644
index 00000000..099e5515
--- /dev/null
+++ b/src/website_profiling/api/routers/chat.py
@@ -0,0 +1,243 @@
+"""Chat routers — /api/chat, /api/chat/sessions/*, /api/chat/artifacts/*."""
+from __future__ import annotations
+
+import json
+import queue
+import re
+import threading
+from typing import Annotated, Any, Generator, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from fastapi.responses import StreamingResponse
+from psycopg import Connection
+
+from ..deps import get_db
+from ..schemas.chat import ArtifactUpdateBody, ChatRequest, ChatSessionCreate
+
+router = APIRouter(prefix="/chat", tags=["chat"])
+
+DbDep = Annotated[Connection, Depends(get_db)]
+
+_FIRST_SENTENCE_RE = re.compile(r"^(.{8,80}[.!?])", re.DOTALL)
+
+
+def _fmt_session(s: dict[str, Any]) -> dict[str, Any]:
+ return {
+ "id": s["id"],
+ "propertyId": s["property_id"],
+ "title": s["title"],
+ "createdAt": s["created_at"],
+ "updatedAt": s["updated_at"],
+ }
+
+
+def _messages_for_agent_context(
+ rows: list[dict[str, Any]], max_turns: int = 20
+) -> list[dict[str, str]]:
+ """Port of messagesForAgentContext from chatDb.ts."""
+ relevant = [m for m in rows if m.get("role") in ("user", "assistant")]
+ sliced = relevant[-(max_turns * 2):]
+ return [{"role": m["role"], "content": str(m.get("content") or "")} for m in sliced]
+
+
+def _derive_title(text: str) -> str | None:
+ text = text.strip()
+ if not text:
+ return None
+ m = _FIRST_SENTENCE_RE.match(text)
+ raw = m.group(1).strip() if m else text[:60].strip()
+ return raw[:80] if raw else None
+
+
+# ── POST /api/chat (SSE streaming) ────────────────────────────────────────────
+
+@router.post("/")
+def chat_turn(body: ChatRequest, conn: DbDep) -> StreamingResponse:
+ from website_profiling.db.chat_store import (
+ append_message,
+ get_messages,
+ get_session,
+ update_session_title,
+ )
+ from website_profiling.llm.agent import run_agent_turn
+ from website_profiling.tools.audit_tools import AuditToolContext
+
+ # Validate session
+ session = get_session(conn, body.sessionId)
+ if not session or session["property_id"] != body.propertyId:
+ raise HTTPException(status_code=404, detail="session not found")
+
+ # Persist user message
+ append_message(conn, body.sessionId, "user", body.message)
+
+ # Build agent context
+ history = get_messages(conn, body.sessionId)
+ agent_messages = _messages_for_agent_context(history)
+ context = AuditToolContext(
+ property_id=body.propertyId,
+ report_id=body.reportId,
+ )
+
+ q: queue.Queue[dict[str, Any] | None] = queue.Queue()
+ assistant_parts: list[str] = []
+ tool_events: list[dict[str, Any]] = []
+ result_holder: list[dict[str, Any]] = []
+
+ def on_event(event: dict[str, Any]) -> None:
+ if event.get("type") == "token":
+ assistant_parts.append(str(event.get("text") or ""))
+ elif event.get("type") == "tool_end":
+ tool_events.append(event)
+ q.put(event)
+
+ def run_agent() -> None:
+ try:
+ result = run_agent_turn(agent_messages, context, on_event=on_event)
+ result_holder.append(result)
+ except Exception as exc:
+ q.put({"type": "error", "message": str(exc)})
+ finally:
+ q.put(None) # sentinel
+
+ thread = threading.Thread(target=run_agent, daemon=True)
+ thread.start()
+
+ def generate() -> Generator[str, None, None]:
+ while True:
+ item = q.get()
+ if item is None:
+ break
+ event_type = str(item.get("type") or "message")
+ yield f"event: {event_type}\ndata: {json.dumps(item)}\n\n"
+
+ thread.join(timeout=5)
+
+ # Persist assistant response. The injected `conn` dependency is released
+ # by FastAPI as soon as StreamingResponse is returned (before this
+ # generator resumes), so we open a fresh connection for persistence.
+ assistant_text = "".join(assistant_parts).strip()
+ if assistant_text:
+ try:
+ from website_profiling.db.pool import db_session
+ with db_session() as fresh_conn:
+ append_message(fresh_conn, body.sessionId, "assistant", assistant_text)
+ if session.get("title") in ("New chat", "", None):
+ derived = _derive_title(body.message) or _derive_title(assistant_text)
+ if derived:
+ update_session_title(fresh_conn, body.sessionId, derived)
+ except Exception as exc:
+ import logging
+ logging.getLogger(__name__).warning(
+ "Failed to persist chat assistant message for session %s: %s",
+ body.sessionId,
+ exc,
+ )
+
+ return StreamingResponse(generate(), media_type="text/event-stream")
+
+
+# ── Session CRUD ──────────────────────────────────────────────────────────────
+
+@router.get("/sessions")
+def list_sessions(
+ conn: DbDep,
+ propertyId: int = Query(...),
+) -> dict[str, Any]:
+ from website_profiling.db.chat_store import list_sessions as _list
+
+ if not propertyId:
+ raise HTTPException(status_code=400, detail="propertyId required")
+ sessions = _list(conn, propertyId)
+ return {"sessions": [_fmt_session(s) for s in sessions]}
+
+
+@router.post("/sessions")
+def create_session(body: ChatSessionCreate, conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.chat_store import create_session as _create
+
+ if not body.propertyId:
+ raise HTTPException(status_code=400, detail="propertyId required")
+ session_id = _create(conn, body.propertyId, body.title)
+ return {"id": session_id, "propertyId": body.propertyId, "title": body.title}
+
+
+@router.get("/sessions/{session_id}")
+def get_session_route(session_id: int, conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.chat_store import get_session
+
+ session = get_session(conn, session_id)
+ if not session:
+ raise HTTPException(status_code=404, detail="session not found")
+ return {"session": _fmt_session(session)}
+
+
+@router.delete("/sessions/{session_id}")
+def delete_session_route(
+ session_id: int,
+ conn: DbDep,
+ propertyId: int = Query(...),
+) -> dict[str, Any]:
+ from website_profiling.db.chat_store import delete_session, get_session
+
+ session = get_session(conn, session_id)
+ if not session or session["property_id"] != propertyId:
+ raise HTTPException(status_code=404, detail="session not found")
+ deleted = delete_session(conn, session_id)
+ if not deleted:
+ raise HTTPException(status_code=404, detail="session not found")
+ return {"ok": True}
+
+
+@router.get("/sessions/{session_id}/messages")
+def get_session_messages(
+ session_id: int,
+ conn: DbDep,
+ propertyId: int = Query(...),
+) -> dict[str, Any]:
+ from website_profiling.db.chat_store import get_messages, get_session
+
+ session = get_session(conn, session_id)
+ if not session or session["property_id"] != propertyId:
+ raise HTTPException(status_code=404, detail="session not found")
+ messages = get_messages(conn, session_id)
+ return {"messages": messages}
+
+
+# ── Artifacts ────────────────────────────────────────────────────────────────
+
+@router.get("/artifacts/{artifact_id}")
+def get_artifact(artifact_id: str) -> Any:
+ import base64
+ import re as _re
+
+ if not _re.match(r"^[a-f0-9\-]{36}$", artifact_id):
+ raise HTTPException(status_code=400, detail="Invalid artifact id")
+
+ try:
+ from website_profiling.tools.export_artifacts import read_artifact_bytes
+
+ result = read_artifact_bytes(artifact_id)
+ except ImportError:
+ raise HTTPException(status_code=500, detail="Artifact module unavailable")
+
+ if not result:
+ raise HTTPException(status_code=404, detail="Artifact not found")
+
+ meta, data = result
+ filename = meta.get("filename") or "export.bin"
+ mime_type = meta.get("mime_type") or "application/octet-stream"
+ ascii_name = re.sub(r'[^\x20-\x7e]', '_', filename)
+ ascii_name = re.sub(r'["\\/]', '_', ascii_name) or "export.bin"
+
+ from fastapi import Response
+
+ return Response(
+ content=data,
+ media_type=mime_type,
+ headers={
+ "Content-Disposition": (
+ f'attachment; filename="{ascii_name}"; '
+ f"filename*=UTF-8''{ascii_name}"
+ ),
+ },
+ )
diff --git a/src/website_profiling/api/routers/compare.py b/src/website_profiling/api/routers/compare.py
new file mode 100644
index 00000000..14264d9f
--- /dev/null
+++ b/src/website_profiling/api/routers/compare.py
@@ -0,0 +1,61 @@
+"""Report comparison export — /api/compare/*."""
+from __future__ import annotations
+
+from typing import Annotated, Optional
+
+from fastapi import APIRouter, Depends, HTTPException
+from fastapi.responses import Response
+from psycopg import Connection
+from pydantic import BaseModel
+
+from ..deps import get_db
+
+router = APIRouter(tags=["compare"])
+
+DbDep = Annotated[Connection, Depends(get_db)]
+
+
+class CompareExportBody(BaseModel):
+ reportIdA: Optional[int] = None
+ reportIdB: Optional[int] = None
+
+
+def _csv_escape(val: str) -> str:
+ if any(c in val for c in ('",\n')):
+ return f'"{val.replace(chr(34), chr(34) + chr(34))}"'
+ return val
+
+
+@router.post("/compare/export")
+def compare_export(body: CompareExportBody, conn: DbDep) -> Response:
+ if not body.reportIdA or not body.reportIdB:
+ raise HTTPException(status_code=400, detail="reportIdA and reportIdB required")
+
+ from website_profiling.db.report_store import read_report_payload
+
+ payload_a = read_report_payload(conn, body.reportIdA)
+ payload_b = read_report_payload(conn, body.reportIdB)
+ if not payload_a or not payload_b:
+ raise HTTPException(status_code=404, detail="One or both reports not found")
+
+ lines = ["Category,Issue Title,Priority,Change\n"]
+ cats_a = {c.get("id") or c.get("name"): c for c in (payload_a.get("categories") or [])}
+ cats_b = {c.get("id") or c.get("name"): c for c in (payload_b.get("categories") or [])}
+ for key in set(list(cats_a.keys()) + list(cats_b.keys())):
+ cat_a = cats_a.get(key) or {}
+ cat_b = cats_b.get(key) or {}
+ issues_a = {i.get("title"): i for i in (cat_a.get("issues") or [])}
+ issues_b = {i.get("title"): i for i in (cat_b.get("issues") or [])}
+ for title in set(list(issues_a.keys()) + list(issues_b.keys())):
+ in_a = title in issues_a
+ in_b = title in issues_b
+ change = "removed" if in_a and not in_b else "added" if not in_a and in_b else "unchanged"
+ priority = (issues_b.get(title) or issues_a.get(title) or {}).get("priority", "")
+ lines.append(f"{_csv_escape(str(key))},{_csv_escape(str(title))},{priority},{change}\n")
+
+ csv_content = "".join(lines)
+ return Response(
+ content=csv_content,
+ media_type="text/csv",
+ headers={"Content-Disposition": "attachment; filename=compare_export.csv"},
+ )
diff --git a/src/website_profiling/api/routers/config.py b/src/website_profiling/api/routers/config.py
new file mode 100644
index 00000000..70c08f5a
--- /dev/null
+++ b/src/website_profiling/api/routers/config.py
@@ -0,0 +1,260 @@
+"""Config routes: pipeline-config, llm-config, secrets, app-settings."""
+from __future__ import annotations
+
+from typing import Annotated, Any, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from psycopg import Connection
+from pydantic import BaseModel
+
+from ..deps import get_db
+
+router = APIRouter(tags=["config"])
+
+_MASK = "*"
+
+
+# ---------------------------------------------------------------------------
+# helpers
+# ---------------------------------------------------------------------------
+
+
+def _mask_secrets(data: dict[str, Any]) -> dict[str, Any]:
+ """Return a copy of *data* with secret-ish values replaced by ``'*'``."""
+ masked: dict[str, Any] = {}
+ for k, v in data.items():
+ val_str = str(v) if v is not None else ""
+ if val_str and (_is_secret_key(k)):
+ masked[k] = _MASK
+ else:
+ masked[k] = v
+ return masked
+
+
+def _is_secret_key(key: str) -> bool:
+ key_lower = key.lower()
+ return (
+ key_lower.endswith("_secret")
+ or key_lower.endswith("_api_key")
+ or key_lower.endswith("_key")
+ or "api_key" in key_lower
+ or "secret" in key_lower
+ or "password" in key_lower
+ or "token" in key_lower
+ )
+
+
+def _read_llm_config_full(conn: Connection) -> list[dict[str, Any]]:
+ from website_profiling.db.config_store import read_llm_config_full
+ return read_llm_config_full(conn)
+
+
+def _read_app_setting(conn: Connection, key: str) -> Optional[str]:
+ from website_profiling.db.config_store import read_app_setting
+ return read_app_setting(conn, key)
+
+
+def _write_app_setting(conn: Connection, key: str, value: str) -> None:
+ from website_profiling.db.config_store import write_app_setting
+ write_app_setting(conn, key, value)
+
+
+# ---------------------------------------------------------------------------
+# pipeline-config
+# ---------------------------------------------------------------------------
+
+
+@router.get("/pipeline-config")
+def get_pipeline_config(conn: Annotated[Connection, Depends(get_db)]) -> dict[str, Any]:
+ from website_profiling.db.config_store import read_pipeline_config
+
+ state, unknown_keys = read_pipeline_config(conn)
+ return {"state": state, "unknownKeys": unknown_keys, "source": "db"}
+
+
+class PipelineConfigBody(BaseModel):
+ state: dict[str, Any]
+ unknownKeys: Optional[list[dict[str, str]]] = None
+
+
+@router.put("/pipeline-config")
+def put_pipeline_config(
+ body: PipelineConfigBody,
+ conn: Annotated[Connection, Depends(get_db)],
+) -> dict[str, Any]:
+ from website_profiling.db.config_store import write_pipeline_config
+
+ coerced: dict[str, str] = {str(k): str(v) for k, v in body.state.items()}
+ unknown_keys: list[dict[str, str]] = body.unknownKeys or []
+ write_pipeline_config(conn, coerced, unknown_keys)
+ return {"ok": True, "source": "db"}
+
+
+# ---------------------------------------------------------------------------
+# llm-config
+# ---------------------------------------------------------------------------
+
+
+@router.get("/llm-config")
+def get_llm_config(conn: Annotated[Connection, Depends(get_db)]) -> dict[str, Any]:
+ rows = _read_llm_config_full(conn)
+ state: dict[str, Any] = {}
+ for row in rows:
+ k = str(row["key"])
+ v = str(row["value"])
+ is_secret = bool(row.get("is_secret"))
+ state[k] = _MASK if (is_secret and v) else v
+ return {"state": state, "source": "db"}
+
+
+class LlmConfigBody(BaseModel):
+ state: dict[str, Any]
+
+
+@router.put("/llm-config")
+def put_llm_config(
+ body: LlmConfigBody,
+ conn: Annotated[Connection, Depends(get_db)],
+) -> dict[str, Any]:
+ from website_profiling.db.config_store import write_llm_config
+
+ # Preserve existing secret values when client sends "*" (masked sentinel)
+ existing_rows = _read_llm_config_full(conn)
+ existing: dict[str, str] = {str(r["key"]): str(r["value"]) for r in existing_rows}
+ existing_secrets: set[str] = {str(r["key"]) for r in existing_rows if r.get("is_secret")}
+
+ entries: dict[str, str] = {}
+ secret_keys: set[str] = set()
+
+ for k, v in body.state.items():
+ val = str(v) if v is not None else ""
+ is_masked_sentinel = val.strip() in (_MASK, "••••") or (
+ val.strip().startswith("*") and len(val.strip()) <= 4
+ )
+ if is_masked_sentinel and k in existing:
+ # Keep original value
+ entries[k] = existing[k]
+ else:
+ entries[k] = val
+
+ if k in existing_secrets or _is_secret_key(k):
+ secret_keys.add(k)
+
+ write_llm_config(conn, entries, secret_keys)
+ return {"ok": True}
+
+
+# ---------------------------------------------------------------------------
+# secrets
+# ---------------------------------------------------------------------------
+
+
+@router.get("/secrets")
+def get_secrets(conn: Annotated[Connection, Depends(get_db)]) -> dict[str, Any]:
+ from website_profiling.db.google_app_store import read_google_app_settings
+
+ llm_rows = _read_llm_config_full(conn)
+ state: dict[str, Any] = {}
+ for row in llm_rows:
+ k = str(row["key"])
+ v = str(row["value"])
+ is_secret = bool(row.get("is_secret")) or _is_secret_key(k)
+ if is_secret and v:
+ state[k] = _MASK
+ state[f"{k}_masked"] = True
+ elif v:
+ state[k] = v
+
+ google = read_google_app_settings(conn)
+ for field in ("client_id", "client_secret", "developer_token", "login_customer_id"):
+ raw = str(google.get(field) or "")
+ if raw:
+ state[f"google_{field}"] = _MASK if _is_secret_key(field) else raw
+ if _is_secret_key(field):
+ state[f"google_{field}_masked"] = True
+ state["google_has_service_account"] = bool(google.get("service_account_json"))
+
+ return {"state": state, "source": "db"}
+
+
+class SecretsBody(BaseModel):
+ state: dict[str, Any]
+
+
+@router.put("/secrets")
+def put_secrets(
+ body: SecretsBody,
+ conn: Annotated[Connection, Depends(get_db)],
+) -> dict[str, Any]:
+ from website_profiling.db.config_store import read_llm_config, write_llm_config
+ from website_profiling.db.google_app_store import read_google_app_settings, save_google_app_settings
+
+ existing_llm = read_llm_config(conn)
+ existing_rows = _read_llm_config_full(conn)
+ existing_secrets_set: set[str] = {str(r["key"]) for r in existing_rows if r.get("is_secret")}
+
+ llm_updates: dict[str, str] = dict(existing_llm)
+ llm_secret_keys: set[str] = set(existing_secrets_set)
+ google_patch: dict[str, Any] = {}
+
+ for k, v in body.state.items():
+ if k.endswith("_masked") or k == "google_has_service_account":
+ continue
+
+ val = str(v) if v is not None else ""
+ is_masked_sentinel = val.strip() in (_MASK, "••••") or (
+ val.strip().startswith("*") and len(val.strip()) <= 4
+ )
+
+ if k.startswith("google_"):
+ field = k[len("google_"):]
+ if field in ("client_id", "client_secret", "developer_token", "login_customer_id"):
+ if not is_masked_sentinel:
+ google_patch[field] = val
+ else:
+ if is_masked_sentinel:
+ # Preserve existing
+ pass
+ else:
+ llm_updates[k] = val
+ if _is_secret_key(k):
+ llm_secret_keys.add(k)
+
+ write_llm_config(conn, llm_updates, llm_secret_keys)
+
+ if google_patch:
+ save_google_app_settings(conn, google_patch)
+
+ return {"ok": True}
+
+
+# ---------------------------------------------------------------------------
+# app-settings
+# ---------------------------------------------------------------------------
+
+
+@router.get("/app-settings")
+def get_app_setting(
+ conn: Annotated[Connection, Depends(get_db)],
+ key: str = Query(..., description="Settings key to retrieve"),
+) -> dict[str, Any]:
+ if not key or not key.strip():
+ raise HTTPException(status_code=400, detail="Missing key query parameter")
+ value = _read_app_setting(conn, key.strip())
+ return {"key": key.strip(), "value": value}
+
+
+class AppSettingBody(BaseModel):
+ key: str
+ value: str
+
+
+@router.put("/app-settings")
+def put_app_setting(
+ body: AppSettingBody,
+ conn: Annotated[Connection, Depends(get_db)],
+) -> dict[str, Any]:
+ if not body.key or not body.key.strip():
+ raise HTTPException(status_code=400, detail="key must not be empty")
+ _write_app_setting(conn, body.key.strip(), body.value)
+ return {"ok": True}
diff --git a/src/website_profiling/api/routers/content.py b/src/website_profiling/api/routers/content.py
new file mode 100644
index 00000000..aa281200
--- /dev/null
+++ b/src/website_profiling/api/routers/content.py
@@ -0,0 +1,257 @@
+"""Content routers — /api/content/* and /api/backlinks/* and /api/content-drafts/*."""
+from __future__ import annotations
+
+from typing import Annotated, Any
+
+from fastapi import APIRouter, Body, Depends, HTTPException, Query
+from psycopg import Connection
+
+from ..deps import get_db
+from website_profiling.db import content_draft_store
+from website_profiling.integrations.google.gsc_links_store import list_backlinks_velocity
+
+router = APIRouter(tags=["content"])
+
+DbDep = Annotated[Connection, Depends(get_db)]
+
+_VALID_WIZARD_STEPS = {"intents", "content_types", "tones", "titles", "outline", "draft", "research"}
+
+
+# ── GET /api/backlinks/velocity ──────────────────────────────────────────────
+
+@router.get("/backlinks/velocity")
+def backlinks_velocity(
+ conn: DbDep,
+ propertyId: int = Query(...),
+) -> dict[str, Any]:
+ if not propertyId:
+ raise HTTPException(status_code=400, detail="propertyId required")
+ return {"snapshots": list_backlinks_velocity(conn, propertyId)}
+
+
+# ── POST /api/backlinks/competitor-import ────────────────────────────────────
+
+@router.post("/backlinks/competitor-import")
+def backlinks_competitor_import(
+ body: dict[str, Any] = Body(default={}),
+) -> dict[str, Any]:
+ competitor = str(body.get("competitor") or "").strip()
+ csv_text = str(body.get("csvText") or "")
+ our_domains = body.get("ourDomains") or []
+
+ if not competitor or not csv_text.strip():
+ raise HTTPException(status_code=400, detail="competitor and csvText required")
+
+ try:
+ from website_profiling.integrations.google.competitor_links import ( # type: ignore[import]
+ parse_referring_domains_from_csv,
+ build_competitor_domain_gap,
+ )
+
+ refs = parse_referring_domains_from_csv(csv_text)
+ gap = build_competitor_domain_gap(set(our_domains), competitor, refs)
+ return {"gap": gap}
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=f"Competitor backlink import failed: {exc}")
+
+
+# ── POST /api/backlinks/third-party-import ───────────────────────────────────
+
+@router.post("/backlinks/third-party-import")
+def backlinks_third_party_import(
+ conn: DbDep,
+ body: dict[str, Any] = Body(default={}),
+) -> dict[str, Any]:
+ property_id = int(body.get("propertyId") or 0)
+ provider = str(body.get("provider") or "moz").strip().lower()
+ csv_text = str(body.get("csvText") or "")
+ our_domains = body.get("ourDomains") or []
+
+ if not property_id or not csv_text.strip():
+ raise HTTPException(status_code=400, detail="propertyId and csvText required")
+ if provider not in ("moz", "majestic"):
+ raise HTTPException(status_code=400, detail="provider must be moz or majestic")
+
+ try:
+ from website_profiling.integrations.links.third_party_csv import ( # type: ignore[import]
+ build_third_party_overlay,
+ )
+ from website_profiling.integrations.google.gsc_links_store import ( # type: ignore[import]
+ import_third_party_links_overlay,
+ )
+
+ overlay = build_third_party_overlay(provider, csv_text, our_domains)
+ result = import_third_party_links_overlay(conn, property_id, overlay)
+ return result # type: ignore[return-value]
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=f"Third-party backlink import failed: {exc}")
+
+
+# ── POST /api/content/analyze ─────────────────────────────────────────────────
+
+@router.post("/content/analyze")
+def content_analyze(
+ body: dict[str, Any] = Body(default={}),
+) -> dict[str, Any]:
+ keyword = str(body.get("keyword") or "").strip()
+ if not keyword:
+ raise HTTPException(status_code=400, detail="keyword required")
+
+ property_id_raw = body.get("propertyId")
+ property_id = int(property_id_raw) if property_id_raw else None
+
+ try:
+ from website_profiling.content_studio.ai_suggest import analyze_content_draft # type: ignore[import]
+
+ analysis = analyze_content_draft(
+ property_id,
+ keyword,
+ body.get("bodyHtml") or "",
+ body.get("titleTag") or "",
+ body.get("metaDescription") or "",
+ body.get("landingUrl") or None,
+ use_ai=bool(body.get("useAi")),
+ refresh=bool(body.get("refresh")),
+ title=body.get("title") or "",
+ )
+ return {"analysis": analysis}
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=f"Content analyze failed: {exc}")
+
+
+# ── POST /api/content/score ───────────────────────────────────────────────────
+
+@router.post("/content/score")
+def content_score(
+ body: dict[str, Any] = Body(default={}),
+) -> dict[str, Any]:
+ keyword = str(body.get("keyword") or "").strip()
+ if not keyword:
+ raise HTTPException(status_code=400, detail="keyword required")
+
+ property_id_raw = body.get("propertyId")
+ property_id = int(property_id_raw) if property_id_raw else None
+
+ try:
+ from website_profiling.content_studio.score import score_content_draft # type: ignore[import]
+
+ score = score_content_draft(
+ property_id,
+ keyword,
+ body.get("bodyHtml") or "",
+ body.get("titleTag") or "",
+ body.get("metaDescription") or "",
+ body.get("landingUrl") or None,
+ )
+ return {"score": score}
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=f"Content score failed: {exc}")
+
+
+# ── POST /api/content/wizard ──────────────────────────────────────────────────
+
+@router.post("/content/wizard")
+def content_wizard(
+ body: dict[str, Any] = Body(default={}),
+) -> dict[str, Any]:
+ step = str(body.get("step") or "").strip()
+ if step not in _VALID_WIZARD_STEPS:
+ raise HTTPException(status_code=400, detail="Invalid wizard step")
+
+ payload = {
+ "keyword": str(body.get("keyword") or "").strip(),
+ "locale": str(body.get("locale") or "en-US"),
+ "intent": str(body.get("intent") or ""),
+ "contentType": str(body.get("contentType") or ""),
+ "tone": str(body.get("tone") or ""),
+ "title": str(body.get("title") or ""),
+ "outline": body.get("outline") if isinstance(body.get("outline"), list) else [],
+ }
+
+ try:
+ from website_profiling.content_studio.wizard import run_wizard_step # type: ignore[import]
+
+ result = run_wizard_step(step, payload)
+ if isinstance(result, dict) and result.get("ok") is False:
+ raise HTTPException(status_code=400, detail=result.get("error") or "Wizard step failed")
+ return {"result": result}
+ except HTTPException:
+ raise
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=f"Wizard step failed: {exc}")
+
+
+# ── GET /api/content-drafts ───────────────────────────────────────────────────
+
+@router.get("/content-drafts")
+def list_content_drafts_route(
+ conn: DbDep,
+ propertyId: int = Query(...),
+) -> dict[str, Any]:
+ if not propertyId:
+ raise HTTPException(status_code=400, detail="propertyId required")
+ return {"drafts": content_draft_store.list_content_drafts(conn, propertyId)}
+
+
+# ── POST /api/content-drafts ──────────────────────────────────────────────────
+
+@router.post("/content-drafts")
+def create_content_draft_route(
+ conn: DbDep,
+ body: dict[str, Any] = Body(default={}),
+) -> dict[str, Any]:
+ property_id = int(body.get("propertyId") or 0)
+ if not property_id:
+ raise HTTPException(status_code=400, detail="propertyId required")
+
+ draft_id = content_draft_store.create_content_draft(
+ conn,
+ property_id,
+ title=str(body.get("title") or "Untitled draft"),
+ target_keyword=str(body.get("target_keyword") or ""),
+ landing_url=str(body.get("landing_url") or "").strip() or None,
+ status=str(body.get("status") or "draft"),
+ body_html=str(body.get("body_html") or ""),
+ title_tag=str(body.get("title_tag") or ""),
+ meta_description=str(body.get("meta_description") or ""),
+ )
+ return {"id": draft_id, "propertyId": property_id}
+
+
+# ── GET /api/content-drafts/{id} ─────────────────────────────────────────────
+
+@router.get("/content-drafts/{draft_id}")
+def get_content_draft_route(conn: DbDep, draft_id: int) -> dict[str, Any]:
+ if not draft_id:
+ raise HTTPException(status_code=400, detail="invalid draft id")
+ draft = content_draft_store.get_content_draft(conn, draft_id)
+ if not draft:
+ raise HTTPException(status_code=404, detail="draft not found")
+ return {"draft": draft}
+
+
+# ── PATCH /api/content-drafts/{id} ───────────────────────────────────────────
+
+@router.patch("/content-drafts/{draft_id}")
+def update_content_draft_route(
+ conn: DbDep,
+ draft_id: int,
+ body: dict[str, Any] = Body(default={}),
+) -> dict[str, Any]:
+ if not draft_id:
+ raise HTTPException(status_code=400, detail="invalid draft id")
+ draft = content_draft_store.update_content_draft(conn, draft_id, body)
+ if not draft:
+ raise HTTPException(status_code=404, detail="draft not found")
+ return {"draft": draft}
+
+
+# ── DELETE /api/content-drafts/{id} ──────────────────────────────────────────
+
+@router.delete("/content-drafts/{draft_id}")
+def delete_content_draft_route(conn: DbDep, draft_id: int) -> dict[str, Any]:
+ if not draft_id:
+ raise HTTPException(status_code=400, detail="invalid draft id")
+ if not content_draft_store.delete_content_draft(conn, draft_id):
+ raise HTTPException(status_code=404, detail="draft not found")
+ return {"ok": True}
diff --git a/src/website_profiling/api/routers/crawl.py b/src/website_profiling/api/routers/crawl.py
new file mode 100644
index 00000000..3638c8c5
--- /dev/null
+++ b/src/website_profiling/api/routers/crawl.py
@@ -0,0 +1,40 @@
+"""Crawl routes: /api/crawl/*"""
+from __future__ import annotations
+
+from typing import Annotated, Any, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from psycopg import Connection
+
+from ..deps import get_db
+
+router = APIRouter(tags=["crawl"])
+
+
+@router.get("/crawl/browser-status")
+def browser_status_check() -> dict[str, Any]:
+ """Return whether Playwright + Chromium are available."""
+ from website_profiling.crawl.fetchers import ensure_browser_deps
+
+ return ensure_browser_deps()
+
+
+@router.get("/crawl/page-html")
+def get_page_html(
+ conn: Annotated[Connection, Depends(get_db)],
+ url: str = Query(..., description="Page URL to retrieve stored HTML for"),
+ crawlRunId: Optional[int] = Query(None, description="Crawl run ID"),
+) -> dict[str, Any]:
+ """Return stored HTML and metadata for a URL within a crawl run."""
+ from website_profiling.db.html_store import read_page_html
+
+ if not crawlRunId:
+ raise HTTPException(status_code=400, detail="crawlRunId is required")
+
+ result = read_page_html(conn, crawlRunId, url)
+ if result is None:
+ raise HTTPException(
+ status_code=404,
+ detail=f"No stored HTML found for url={url!r} in crawlRunId={crawlRunId}",
+ )
+ return result
diff --git a/src/website_profiling/api/routers/dashboards.py b/src/website_profiling/api/routers/dashboards.py
new file mode 100644
index 00000000..095767b5
--- /dev/null
+++ b/src/website_profiling/api/routers/dashboards.py
@@ -0,0 +1,147 @@
+"""Dashboards router — /api/dashboards/*"""
+from __future__ import annotations
+
+from typing import Annotated, Any, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+from psycopg import Connection
+
+from ..deps import get_db
+from website_profiling.db import dashboard_store
+
+router = APIRouter(tags=["dashboards"])
+
+DbDep = Annotated[Connection, Depends(get_db)]
+
+
+class DashboardCreateBody(BaseModel):
+ propertyId: int
+ name: Optional[str] = None
+ layoutJson: Optional[Any] = None
+
+
+class DashboardUpdateBody(BaseModel):
+ propertyId: int
+ name: Optional[str] = None
+ layoutJson: Optional[Any] = None
+ isDefault: Optional[bool] = None
+
+
+@router.get("/dashboards")
+def list_dashboards(
+ conn: DbDep,
+ propertyId: int = Query(..., description="Property ID"),
+) -> dict[str, Any]:
+ return {"dashboards": dashboard_store.list_dashboards(conn, propertyId)}
+
+
+@router.post("/dashboards", status_code=201)
+def create_dashboard(body: DashboardCreateBody, conn: DbDep) -> dict[str, Any]:
+ name = (body.name or "Untitled dashboard").strip() or "Untitled dashboard"
+ layout = body.layoutJson if body.layoutJson is not None else {}
+ dashboard = dashboard_store.create_dashboard(conn, body.propertyId, name, layout)
+ return {"dashboard": dashboard}
+
+
+@router.get("/dashboards/{dashboard_id}")
+def get_dashboard(
+ dashboard_id: int,
+ conn: DbDep,
+ propertyId: int = Query(..., description="Property ID"),
+) -> dict[str, Any]:
+ dashboard = dashboard_store.get_dashboard(conn, dashboard_id, propertyId)
+ if not dashboard:
+ raise HTTPException(status_code=404, detail="Not found")
+ return {"dashboard": dashboard}
+
+
+@router.put("/dashboards/{dashboard_id}")
+def update_dashboard(dashboard_id: int, body: DashboardUpdateBody, conn: DbDep) -> dict[str, Any]:
+ dashboard = dashboard_store.update_dashboard(
+ conn,
+ dashboard_id,
+ body.propertyId,
+ name=body.name.strip() if body.name is not None else None,
+ layout_json=body.layoutJson,
+ is_default=body.isDefault,
+ )
+ if not dashboard:
+ raise HTTPException(status_code=404, detail="Not found")
+ return {"dashboard": dashboard}
+
+
+@router.delete("/dashboards/{dashboard_id}")
+def delete_dashboard(
+ dashboard_id: int,
+ conn: DbDep,
+ propertyId: int = Query(..., description="Property ID"),
+) -> dict[str, Any]:
+ if not dashboard_store.delete_dashboard(conn, dashboard_id, propertyId):
+ raise HTTPException(status_code=404, detail="Not found")
+ return {"ok": True}
+
+
+class DashboardAiGenerateBody(BaseModel):
+ mode: str
+ prompt: str
+ catalog: list[dict[str, Any]]
+ viz_types: dict[str, str]
+ dashscript_help: str
+ toolName: Optional[str] = None
+ propertyId: Optional[int] = None
+ reportId: Optional[int] = None
+ current: Optional[Any] = None
+ sample: Optional[dict[str, Any]] = None
+
+
+def _truncate_tool_sample(data: dict[str, Any]) -> dict[str, Any]:
+ out: dict[str, Any] = {}
+ for key, val in data.items():
+ out[key] = val[:2] if isinstance(val, list) else val
+ return out
+
+
+@router.post("/dashboards/ai-generate")
+def dashboards_ai_generate(body: DashboardAiGenerateBody, conn: DbDep) -> JSONResponse:
+ """Generate DashScript, a widget, or a full dashboard via LLM."""
+ mode = str(body.mode or "widget").strip().lower()
+ if mode not in {"script", "widget", "dashboard"}:
+ raise HTTPException(status_code=400, detail="mode must be script, widget, or dashboard")
+ prompt = str(body.prompt or "").strip()
+ if not prompt:
+ raise HTTPException(status_code=400, detail="prompt required")
+
+ payload: dict[str, Any] = {
+ "mode": mode,
+ "prompt": prompt,
+ "catalog": body.catalog,
+ "viz_types": body.viz_types,
+ "dashscript_help": body.dashscript_help,
+ "current": body.current,
+ }
+
+ if body.sample is not None:
+ payload["sample"] = body.sample
+ elif body.toolName and body.propertyId and mode in ("script", "widget"):
+ try:
+ from website_profiling.tools.audit_tools import AuditToolContext
+ from website_profiling.tools.audit_tools.registry import dispatch_tool
+
+ ctx = AuditToolContext(property_id=body.propertyId, report_id=body.reportId)
+ tool_result = dispatch_tool(body.toolName, {}, context=ctx, conn=conn)
+ if isinstance(tool_result, dict) and "error" not in tool_result:
+ payload["sample"] = _truncate_tool_sample(tool_result)
+ except Exception:
+ pass
+
+ from website_profiling.db.config_store import read_llm_config
+ from website_profiling.llm.dashboard_ai import generate_dashboard_ai
+
+ cfg = read_llm_config(conn)
+ result = generate_dashboard_ai(payload, cfg=cfg or None)
+ if result.get("ok") is False:
+ status = 503 if result.get("missing") else 500
+ return JSONResponse(content=result, status_code=status)
+ return JSONResponse(content=result)
diff --git a/src/website_profiling/api/routers/filters.py b/src/website_profiling/api/routers/filters.py
new file mode 100644
index 00000000..b2fb19d7
--- /dev/null
+++ b/src/website_profiling/api/routers/filters.py
@@ -0,0 +1,55 @@
+"""Saved filters router — /api/filters"""
+from __future__ import annotations
+
+from typing import Annotated, Any, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from pydantic import BaseModel
+from psycopg import Connection
+
+from ..deps import get_db
+from website_profiling.db import saved_filter_store
+
+router = APIRouter(tags=["filters"])
+
+DbDep = Annotated[Connection, Depends(get_db)]
+
+
+class FilterUpsertBody(BaseModel):
+ propertyId: int
+ name: str
+ filterJson: Optional[Any] = None
+
+
+class FilterDeleteBody(BaseModel):
+ propertyId: int
+ name: str
+
+
+@router.get("/filters")
+def list_filters(
+ conn: DbDep,
+ propertyId: int = Query(..., description="Property ID"),
+) -> dict[str, Any]:
+ return {"filters": saved_filter_store.list_saved_filters(conn, propertyId)}
+
+
+@router.post("/filters")
+def upsert_filter(body: FilterUpsertBody, conn: DbDep) -> dict[str, Any]:
+ name = (body.name or "").strip()
+ if not body.propertyId or not name:
+ raise HTTPException(status_code=400, detail="propertyId and name required")
+ filter_json = body.filterJson if isinstance(body.filterJson, dict) else {}
+ saved_filter_store.upsert_saved_filter(conn, body.propertyId, name, filter_json)
+ return {"ok": True}
+
+
+@router.delete("/filters")
+def delete_filter(body: FilterDeleteBody, conn: DbDep) -> dict[str, Any]:
+ name = (body.name or "").strip()
+ if not body.propertyId or not name:
+ raise HTTPException(status_code=400, detail="propertyId and name required")
+ deleted = saved_filter_store.delete_saved_filter(conn, body.propertyId, name)
+ if not deleted:
+ raise HTTPException(status_code=404, detail="filter not found")
+ return {"ok": True}
diff --git a/src/website_profiling/api/routers/health.py b/src/website_profiling/api/routers/health.py
new file mode 100644
index 00000000..fe00d023
--- /dev/null
+++ b/src/website_profiling/api/routers/health.py
@@ -0,0 +1,17 @@
+"""GET /api/health — liveness + DB check."""
+from __future__ import annotations
+
+from typing import Annotated
+
+from fastapi import APIRouter, Depends
+from psycopg import Connection
+
+from ..deps import get_db
+
+router = APIRouter(tags=["health"])
+
+
+@router.get("/health")
+def health_check(conn: Annotated[Connection, Depends(get_db)]) -> dict:
+ conn.execute("SELECT 1")
+ return {"ok": True, "database": "up"}
diff --git a/src/website_profiling/api/routers/integrations.py b/src/website_profiling/api/routers/integrations.py
new file mode 100644
index 00000000..c8a6fe1d
--- /dev/null
+++ b/src/website_profiling/api/routers/integrations.py
@@ -0,0 +1,549 @@
+"""Integrations routers — /api/integrations/google/* and /api/integrations/bing/*."""
+from __future__ import annotations
+
+import json
+import sys
+from typing import Annotated, Any, Optional
+
+from fastapi import APIRouter, Body, Depends, HTTPException, Query
+from psycopg import Connection
+
+from ..deps import get_db
+
+router = APIRouter(prefix="/integrations", tags=["integrations"])
+
+DbDep = Annotated[Connection, Depends(get_db)]
+
+# ── Helpers ────────────────────────────────────────────────────────────────────
+
+def _google_public_status(conn: Connection) -> dict[str, Any]:
+ """Build a public status dict from google_app_settings."""
+ from website_profiling.db.google_app_store import read_google_app_settings
+
+ cfg = read_google_app_settings(conn)
+ has_client_id = bool(cfg.get("client_id"))
+ has_client_secret = bool(cfg.get("client_secret"))
+ has_service_account = bool(cfg.get("service_account_json"))
+ sa = cfg.get("service_account_json") or {}
+ return {
+ "hasClientId": has_client_id,
+ "hasClientSecret": has_client_secret,
+ "hasOAuthApp": has_client_id and has_client_secret,
+ "hasServiceAccount": has_service_account,
+ "serviceAccountEmail": sa.get("client_email") if has_service_account else None,
+ "dateRangeDays": cfg.get("default_date_range_days", 28),
+ "hasDeveloperToken": bool(cfg.get("developer_token")),
+ "hasLoginCustomerId": bool(cfg.get("login_customer_id")),
+ }
+
+
+# ── GET /api/integrations/google/credentials ──────────────────────────────────
+
+@router.get("/google/credentials")
+def get_google_credentials(conn: DbDep) -> dict[str, Any]:
+ """Full app-level Google OAuth settings (server-side / local admin only)."""
+ from website_profiling.db.google_app_store import read_google_app_settings
+
+ cfg = read_google_app_settings(conn)
+ sa = cfg.get("service_account_json")
+ return {
+ "clientId": str(cfg.get("client_id") or "").strip(),
+ "clientSecret": str(cfg.get("client_secret") or "").strip(),
+ "serviceAccount": sa if isinstance(sa, dict) else None,
+ "dateRangeDays": int(cfg.get("default_date_range_days") or 28),
+ "developerToken": str(cfg.get("developer_token") or "").strip(),
+ "loginCustomerId": str(cfg.get("login_customer_id") or "").strip(),
+ }
+
+
+# ── GET /api/integrations/google/status ───────────────────────────────────────
+
+@router.get("/google/status")
+def google_status(conn: DbDep) -> dict[str, Any]:
+ from website_profiling.integrations.google.store import read_last_google_fetched_at
+
+ status = _google_public_status(conn)
+ status["lastFetchedAt"] = read_last_google_fetched_at(conn)
+ return status
+
+
+# ── POST /api/integrations/google/credentials ─────────────────────────────────
+
+@router.post("/google/credentials")
+def save_google_credentials(
+ conn: DbDep,
+ body: dict[str, Any] = Body(default={}),
+) -> dict[str, Any]:
+ _PROPERTY_ONLY_MSG = (
+ "Per-site settings (GSC, GA4, refresh token) must be saved via property "
+ "Integrations when a Site URL is set."
+ )
+ if any(k in body for k in ("refreshToken", "gscSiteUrl", "ga4PropertyId")):
+ raise HTTPException(status_code=400, detail=_PROPERTY_ONLY_MSG)
+
+ from website_profiling.db.google_app_store import save_google_app_settings
+
+ patch: dict[str, Any] = {}
+ if isinstance(body.get("clientId"), str) and body["clientId"].strip():
+ patch["client_id"] = body["clientId"].strip()
+ if isinstance(body.get("clientSecret"), str) and body["clientSecret"].strip():
+ patch["client_secret"] = body["clientSecret"].strip()
+ if isinstance(body.get("dateRangeDays"), (int, float)) and body["dateRangeDays"] > 0:
+ patch["default_date_range_days"] = int(body["dateRangeDays"])
+ if isinstance(body.get("developerToken"), str) and body["developerToken"].strip():
+ patch["developer_token"] = body["developerToken"].strip()
+ if isinstance(body.get("loginCustomerId"), str) and body["loginCustomerId"].strip():
+ patch["login_customer_id"] = body["loginCustomerId"].strip().replace("-", "")
+
+ if not patch:
+ raise HTTPException(status_code=400, detail="No valid fields provided")
+
+ save_google_app_settings(conn, patch)
+ return {"ok": True, "status": _google_public_status(conn)}
+
+
+# ── POST /api/integrations/google/credentials/upload ──────────────────────────
+
+@router.post("/google/credentials/upload")
+def upload_google_credentials(
+ conn: DbDep,
+ body: dict[str, Any] = Body(default={}),
+) -> dict[str, Any]:
+ from website_profiling.db.google_app_store import save_google_app_settings
+
+ raw = body.get("fileContent")
+ if not raw or not isinstance(raw, str):
+ raise HTTPException(status_code=400, detail="fileContent is required")
+
+ try:
+ parsed = json.loads(raw)
+ except Exception:
+ raise HTTPException(status_code=400, detail="This doesn't look like a valid JSON file.")
+
+ if (
+ not isinstance(parsed, dict)
+ or parsed.get("type") != "service_account"
+ or not isinstance(parsed.get("client_email"), str)
+ or not isinstance(parsed.get("private_key"), str)
+ ):
+ raise HTTPException(
+ status_code=400,
+ detail=(
+ "This doesn't look like a Google service account key file. "
+ "Make sure you downloaded the JSON key from Google Cloud Console > "
+ "IAM & Admin > Service Accounts."
+ ),
+ )
+
+ save_google_app_settings(conn, {"service_account_json": parsed})
+ return {"ok": True, "status": _google_public_status(conn)}
+
+
+# ── POST /api/integrations/google/disconnect ──────────────────────────────────
+
+@router.post("/google/disconnect")
+def google_disconnect(conn: DbDep) -> dict[str, Any]:
+ """Global disconnect is deprecated — use per-property disconnect."""
+ return {
+ "ok": False,
+ "error": (
+ "Disconnect Google per site: set Site URL, open Integrations, "
+ "and use Disconnect on that property."
+ ),
+ "status": _google_public_status(conn),
+ }
+
+
+# ── GET /api/integrations/google/properties ───────────────────────────────────
+
+@router.get("/google/properties")
+def google_properties_deprecated(
+ property_id: Optional[int] = Query(None, alias="propertyId"),
+) -> dict[str, Any]:
+ """Deprecated — use /api/properties/{id}/google/properties."""
+ if not property_id:
+ raise HTTPException(
+ status_code=400,
+ detail="propertyId query parameter is required. Use /api/properties/{id}/google/properties instead.",
+ )
+ raise HTTPException(
+ status_code=301,
+ detail=f"Use /api/properties/{property_id}/google/properties",
+ )
+
+
+# ── POST /api/integrations/google/test ────────────────────────────────────────
+
+@router.post("/google/test")
+def google_test() -> dict[str, Any]:
+ """Run `python -m src google --test` and return stdout log."""
+ import subprocess
+ import sys
+
+ try:
+ result = subprocess.run(
+ [sys.executable, "-m", "src", "google", "--test"],
+ capture_output=True,
+ text=True,
+ timeout=30,
+ )
+ log = (result.stdout + result.stderr)[-28_000:]
+ return {"ok": result.returncode == 0, "log": log, "exitCode": result.returncode}
+ except subprocess.TimeoutExpired:
+ return {"ok": False, "log": "", "error": "Test timed out after 30s"}
+ except Exception as exc:
+ return {"ok": False, "log": "", "error": str(exc)}
+
+
+# ── GET /api/integrations/google/page-data ────────────────────────────────────
+
+@router.get("/google/page-data")
+def google_page_data(
+ conn: DbDep,
+ url: str = Query(...),
+ googleSnapshotId: Optional[int] = Query(None),
+ propertyId: Optional[str] = Query(None),
+ domain: Optional[str] = Query(None),
+) -> dict[str, Any]:
+ from website_profiling.db.property_store import resolve_property_id_for_page
+ from website_profiling.integrations.google.page_lookup import slice_from_google_row
+ from website_profiling.integrations.google.store import read_google_snapshot_row
+
+ if not url:
+ raise HTTPException(status_code=400, detail="url parameter required")
+
+ property_id = resolve_property_id_for_page(conn, url, propertyId, domain)
+
+ _empty = {
+ "source": "snapshot",
+ "snapshotId": None,
+ "gsc": None,
+ "ga4": None,
+ "coverage": {"inCrawl": False, "inGsc": False, "inGa4": False},
+ "siteBenchmarks": {"gsc": None, "ga4": None},
+ "dateRange": {},
+ "fetchedAt": None,
+ }
+
+ if property_id is None:
+ return _empty
+
+ snap = read_google_snapshot_row(
+ conn,
+ property_id,
+ snapshot_id=googleSnapshotId,
+ )
+ if not snap:
+ return _empty
+
+ slice_data = slice_from_google_row(snap["data"], url)
+ return {
+ **slice_data,
+ "snapshotId": snap["id"],
+ "fetchedAt": snap["fetchedAt"] or slice_data.get("fetchedAt"),
+ }
+
+
+# ── GET /api/integrations/google/page-data/history ────────────────────────────
+
+@router.get("/google/page-data/history")
+def google_page_data_history(
+ conn: DbDep,
+ url: str = Query(...),
+ propertyId: Optional[str] = Query(None),
+ domain: Optional[str] = Query(None),
+) -> dict[str, Any]:
+ from website_profiling.db.property_store import resolve_property_id_for_page
+ from website_profiling.integrations.google.page_lookup import (
+ slice_from_google_row,
+ summary_from_slice,
+ )
+ from website_profiling.integrations.google.store import list_google_snapshot_rows
+
+ if not url:
+ raise HTTPException(status_code=400, detail="url parameter required")
+
+ property_id = resolve_property_id_for_page(conn, url, propertyId, domain)
+ if property_id is None:
+ return {"url": url, "history": []}
+
+ history: list[dict[str, Any]] = []
+ for snap in list_google_snapshot_rows(conn, property_id, limit=10):
+ slice_data = slice_from_google_row(snap["data"], url)
+ if not slice_data.get("gsc") and not slice_data.get("ga4"):
+ continue
+ summary = summary_from_slice(slice_data.get("gsc"), slice_data.get("ga4"))
+ history.append({
+ "id": snap["id"],
+ "fetchedAt": snap["fetchedAt"],
+ "type": "snapshot",
+ "gsc": summary.get("gsc"),
+ "ga4": summary.get("ga4"),
+ })
+
+ return {"url": url, "history": history}
+
+
+# ── POST /api/integrations/google/page-live ───────────────────────────────────
+
+@router.post("/google/page-live")
+def google_page_live(
+ body: dict[str, Any] = Body(default={}),
+) -> dict[str, Any]:
+ url = str(body.get("url") or "").strip()
+ if not url:
+ raise HTTPException(status_code=400, detail="url is required")
+
+ try:
+ result = subprocess.run(
+ [sys.executable, "-m", "src", "page-live", "--url", url],
+ capture_output=True,
+ text=True,
+ timeout=45,
+ )
+ combined = result.stdout + result.stderr
+ log = combined[-28_000:]
+ lines = [ln for ln in result.stdout.strip().splitlines() if ln]
+ last = lines[-1] if lines else "{}"
+ try:
+ data = json.loads(last)
+ except Exception:
+ data = {}
+
+ if result.returncode != 0 and not data.get("ok") and not data.get("gsc") and not data.get("ga4"):
+ raise HTTPException(
+ status_code=500,
+ detail=data.get("error") or "Live fetch failed",
+ )
+ import datetime
+ return {"ok": True, "fetchedAt": datetime.datetime.now(datetime.timezone.utc).isoformat().replace("+00:00", "Z"), **data}
+ except subprocess.TimeoutExpired:
+ raise HTTPException(status_code=504, detail="Live fetch timed out after 45s")
+
+
+# ── GET /api/integrations/google/keywords/by-page ─────────────────────────────
+
+@router.get("/google/keywords/by-page")
+def google_keywords_by_page(
+ conn: DbDep,
+ url: str = Query(..., alias="url"),
+ propertyId: Optional[str] = Query(None),
+ domain: Optional[str] = Query(None),
+) -> dict[str, Any]:
+ from website_profiling.db.property_store import resolve_property_id_for_page
+ from website_profiling.integrations.google.keyword_store import read_latest_keyword_data
+
+ page_url = url.strip()
+ if not page_url:
+ raise HTTPException(status_code=400, detail="url parameter is required")
+
+ property_id = resolve_property_id_for_page(conn, page_url, propertyId, domain)
+ if property_id is None:
+ raise HTTPException(status_code=400, detail="propertyId or domain required")
+
+ data = read_latest_keyword_data(conn, property_id) or {}
+ all_rows = data.get("rows") or []
+ normalized_target = page_url.lower().rstrip("/")
+
+ page_keywords = [
+ r for r in all_rows
+ if _matches_url(r.get("gsc_url") or "", normalized_target)
+ ]
+
+ cannib_raw = data.get("cannibalisation") or []
+ cannib = [
+ c for c in cannib_raw
+ if any(
+ (p.get("url") or "").lower().rstrip("/") == normalized_target
+ for p in (c.get("pages") or [])
+ )
+ ]
+
+ return {
+ "url": page_url,
+ "propertyId": property_id,
+ "keyword_count": len(page_keywords),
+ "keywords": page_keywords,
+ "cannibalisation": cannib,
+ "fetched_at": data.get("fetched_at"),
+ }
+
+
+def _matches_url(candidate: str, target: str) -> bool:
+ u = candidate.lower().rstrip("/")
+ return u == target or u in target or target in u
+
+
+# ── GET /api/integrations/google/keywords/history ────────────────────────────
+
+@router.get("/google/keywords/history")
+def google_keywords_history(
+ conn: DbDep,
+ keyword: str = Query(...),
+ propertyId: Optional[str] = Query(None),
+ domain: Optional[str] = Query(None),
+ limit: int = Query(30, ge=1, le=90),
+) -> dict[str, Any]:
+ from website_profiling.db.property_store import resolve_property_id_for_page
+ from website_profiling.integrations.google.keyword_store import read_keyword_history
+
+ keyword = keyword.strip()
+ if not keyword:
+ raise HTTPException(status_code=400, detail="keyword parameter is required")
+
+ property_id = resolve_property_id_for_page(conn, "", propertyId, domain)
+ if property_id is None:
+ raise HTTPException(status_code=400, detail="propertyId or domain required")
+
+ history = read_keyword_history(conn, keyword, limit, property_id=property_id)
+ return {"keyword": keyword, "propertyId": property_id, "history": history}
+
+
+# ── POST /api/integrations/bing/sync ─────────────────────────────────────────
+
+@router.post("/bing/sync")
+def bing_sync(conn: DbDep) -> dict[str, Any]:
+ """Fetch Bing Webmaster backlinks summary using config from DB."""
+ from website_profiling.db.config_store import read_pipeline_config
+
+ try:
+ state, _ = read_pipeline_config(conn)
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
+
+ api_key = str(state.get("bing_webmaster_api_key") or "").strip()
+ site_url = str(state.get("start_url") or "").strip()
+
+ if not api_key or not site_url:
+ raise HTTPException(
+ status_code=400,
+ detail="Set bing_webmaster_api_key and start_url in pipeline settings.",
+ )
+
+ try:
+ from website_profiling.integrations.bing.webmaster import fetch_bing_backlinks_summary
+
+ result = fetch_bing_backlinks_summary(api_key, site_url)
+ return result # type: ignore[return-value]
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
+
+
+# ── GET /api/integrations/google/page-compare ────────────────────────────────
+
+@router.get("/google/page-compare")
+def google_page_compare(
+ conn: DbDep,
+ url: str = Query(...),
+ currentType: str = Query("snapshot"),
+ currentId: int = Query(...),
+ baselineType: str = Query("snapshot"),
+ baselineId: int = Query(...),
+) -> dict[str, Any]:
+ """Compare two page Google data snapshots."""
+ from website_profiling.integrations.google.page_snapshot_store import read_page_snapshot_compare
+
+ current = read_page_snapshot_compare(conn, currentId)
+ baseline = read_page_snapshot_compare(conn, baselineId)
+ if current is None:
+ raise HTTPException(status_code=404, detail="Current snapshot not found")
+ if baseline is None:
+ raise HTTPException(status_code=404, detail="Baseline snapshot not found")
+ return {"url": url, "current": current, "baseline": baseline}
+
+
+# ── GET /api/integrations/google/page-live/history ────────────────────────────
+
+@router.get("/google/page-live/history")
+def google_page_live_history(
+ conn: DbDep,
+ url: str = Query(...),
+ limit: int = Query(15, ge=1, le=50),
+) -> dict[str, Any]:
+ """Return history of page Google snapshots for a URL."""
+ from website_profiling.integrations.google.page_snapshot_store import list_page_snapshot_api_history
+
+ try:
+ history = list_page_snapshot_api_history(conn, url, limit=limit)
+ return {"url": url, "history": history}
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
+
+
+# ── POST /api/integrations/google/keywords/history/batch ─────────────────────
+
+@router.post("/google/keywords/history/batch")
+def google_keywords_history_batch(
+ conn: DbDep,
+ body: dict[str, Any],
+) -> dict[str, Any]:
+ """Batch keyword history: { keywords: str[], limit?: int, propertyId?: int, domain?: str }"""
+ from website_profiling.db.property_store import get_property_id_by_domain
+ from website_profiling.integrations.google.keyword_store import read_keyword_history_batch
+
+ keywords_raw = body.get("keywords") or []
+ if not isinstance(keywords_raw, list):
+ raise HTTPException(status_code=400, detail="keywords must be a list")
+ keywords = [str(k).strip() for k in keywords_raw[:100] if k]
+ limit = max(1, min(int(body.get("limit") or 30), 90))
+ property_id = None
+ if body.get("propertyId"):
+ try:
+ property_id = int(body["propertyId"])
+ except (TypeError, ValueError):
+ pass
+ elif body.get("domain"):
+ property_id = get_property_id_by_domain(conn, str(body["domain"]))
+
+ if property_id is None:
+ raise HTTPException(status_code=400, detail="propertyId or domain required")
+
+ results = read_keyword_history_batch(
+ conn,
+ keywords,
+ property_id=property_id,
+ limit=limit,
+ )
+ return {"keywords": results, "propertyId": property_id}
+
+
+# ── GET/POST /api/integrations/google/keywords/expand ────────────────────────
+
+@router.post("/google/keywords/expand")
+def google_keywords_expand(
+ conn: DbDep,
+ body: dict[str, Any],
+) -> dict[str, Any]:
+ """Expand keyword ideas from Google Keyword Planner or suggest API."""
+ keyword = str(body.get("keyword") or "").strip()
+ if not keyword:
+ raise HTTPException(status_code=400, detail="keyword required")
+ try:
+ from website_profiling.tools.keyword_suggestions import expand_keyword
+ result = expand_keyword(keyword, body.get("propertyId"), conn)
+ return result if isinstance(result, dict) else {"keywords": result}
+ except ImportError:
+ raise HTTPException(status_code=501, detail="Keyword expansion unavailable")
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
+
+
+# ── POST /api/integrations/google/keywords/planner ────────────────────────────
+
+@router.post("/google/keywords/planner")
+def google_keywords_planner(
+ conn: DbDep,
+ body: dict[str, Any],
+) -> dict[str, Any]:
+ """Fetch keyword planner data from Google Ads API."""
+ keywords_raw = body.get("keywords") or []
+ if not isinstance(keywords_raw, list):
+ raise HTTPException(status_code=400, detail="keywords must be a list")
+ try:
+ from website_profiling.integrations.google.keyword_planner import fetch_keyword_ideas
+ result = fetch_keyword_ideas(conn, keywords_raw)
+ return result if isinstance(result, dict) else {"ideas": result}
+ except ImportError:
+ raise HTTPException(status_code=501, detail="Google Keyword Planner unavailable")
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
diff --git a/src/website_profiling/api/routers/issues.py b/src/website_profiling/api/routers/issues.py
new file mode 100644
index 00000000..1f405ef8
--- /dev/null
+++ b/src/website_profiling/api/routers/issues.py
@@ -0,0 +1,148 @@
+"""Issues routers — /api/issues/* and /api/ai/*."""
+from __future__ import annotations
+
+from typing import Annotated, Any
+
+from fastapi import APIRouter, Body, Depends, HTTPException, Query
+from psycopg import Connection
+
+from ..deps import get_db
+from website_profiling.db import issue_status_store
+
+router = APIRouter(tags=["issues"])
+
+DbDep = Annotated[Connection, Depends(get_db)]
+
+
+# ── GET /api/issues/status ────────────────────────────────────────────────────
+
+@router.get("/issues/status")
+def list_issue_status_route(
+ conn: DbDep,
+ propertyId: int = Query(...),
+) -> dict[str, Any]:
+ if not propertyId:
+ raise HTTPException(status_code=400, detail="propertyId required")
+ return {"issues": issue_status_store.list_issue_status(conn, propertyId)}
+
+
+# ── PUT /api/issues/status ────────────────────────────────────────────────────
+
+@router.put("/issues/status")
+def upsert_issue_status_route(
+ conn: DbDep,
+ body: dict[str, Any] = Body(default={}),
+) -> dict[str, Any]:
+ property_id = int(body.get("propertyId") or 0)
+ message = str(body.get("message") or "").strip()
+ status = str(body.get("status") or "")
+
+ if not property_id or not message or not status:
+ raise HTTPException(
+ status_code=400,
+ detail="propertyId, message, and valid status required",
+ )
+
+ report_id = body.get("reportId")
+ try:
+ issue = issue_status_store.upsert_issue_status(
+ conn,
+ property_id=property_id,
+ message=message,
+ status=status,
+ report_id=int(report_id) if report_id is not None else None,
+ url=str(body.get("url") or ""),
+ priority=str(body.get("priority") or "Medium"),
+ category_id=body.get("categoryId") or None,
+ assignee=body.get("assignee") or None,
+ note=body.get("note") or None,
+ )
+ except ValueError as exc:
+ raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+ return {"issue": issue}
+
+
+# ── POST /api/issues/fix-suggestion ──────────────────────────────────────────
+
+@router.post("/issues/fix-suggestion")
+def issues_fix_suggestion(
+ body: dict[str, Any] = Body(default={}),
+) -> Any:
+ message = str(body.get("message") or "").strip()
+ if not message:
+ raise HTTPException(status_code=400, detail="message required")
+
+ payload = {
+ "source": "issue",
+ "message": message,
+ "url": body.get("url"),
+ "priority": body.get("priority"),
+ "category": body.get("category"),
+ "recommendation": body.get("recommendation"),
+ "type": body.get("type"),
+ "refresh": body.get("refresh"),
+ }
+
+ try:
+ from website_profiling.llm.fix_suggestions import generate_fix_suggestion # type: ignore[import]
+
+ return generate_fix_suggestion(payload, refresh=bool(payload.get("refresh")))
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=f"Fix suggestion failed: {exc}")
+
+
+# ── POST /api/issues/action-plan ──────────────────────────────────────────────
+
+@router.post("/issues/action-plan")
+def issues_action_plan(
+ body: dict[str, Any] = Body(default={}),
+) -> Any:
+ domain = str(body.get("domain") or "").strip()
+ if not domain:
+ raise HTTPException(status_code=400, detail="domain required")
+ if not isinstance(body.get("issues"), list) or len(body["issues"]) == 0:
+ raise HTTPException(status_code=400, detail="issues required")
+
+ payload = {
+ "domain": domain,
+ "issues": body["issues"],
+ "refresh": body.get("refresh"),
+ }
+
+ try:
+ from website_profiling.llm.issues_action_plan import generate_issues_action_plan # type: ignore[import]
+
+ return generate_issues_action_plan(payload, refresh=bool(payload.get("refresh")))
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=f"Action plan failed: {exc}")
+
+
+# ── POST /api/ai/fix-suggestion ──────────────────────────────────────────────
+
+@router.post("/ai/fix-suggestion")
+def ai_fix_suggestion(
+ body: dict[str, Any] = Body(default={}),
+) -> Any:
+ message = str(body.get("message") or "").strip()
+ if not message:
+ raise HTTPException(status_code=400, detail="message required")
+
+ payload = {
+ "source": body.get("source") or "issue",
+ "message": message,
+ "url": body.get("url"),
+ "refresh": body.get("refresh"),
+ "context": body.get("context"),
+ "priority": body.get("priority"),
+ "category": body.get("category"),
+ "recommendation": body.get("recommendation"),
+ "type": body.get("type"),
+ }
+
+ try:
+ from website_profiling.llm.fix_suggestions import generate_fix_suggestion # type: ignore[import]
+
+ return generate_fix_suggestion(payload, refresh=bool(payload.get("refresh")))
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=f"Fix suggestion failed: {exc}")
diff --git a/src/website_profiling/api/routers/keywords.py b/src/website_profiling/api/routers/keywords.py
new file mode 100644
index 00000000..4c676cf0
--- /dev/null
+++ b/src/website_profiling/api/routers/keywords.py
@@ -0,0 +1,72 @@
+"""Keywords routers — /api/keywords/*."""
+from __future__ import annotations
+
+from typing import Annotated, Any, Optional
+
+from fastapi import APIRouter, Body, Depends, HTTPException
+
+from ..deps import get_db
+from psycopg import Connection
+
+router = APIRouter(prefix="/keywords", tags=["keywords"])
+
+DbDep = Annotated[Connection, Depends(get_db)]
+
+
+# ── POST /api/keywords/competitor-import ──────────────────────────────────────
+
+@router.post("/competitor-import")
+def keywords_competitor_import(
+ conn: DbDep,
+ body: dict[str, Any] = Body(default={}),
+) -> dict[str, Any]:
+ property_id = int(body.get("propertyId") or 0)
+ competitor = str(body.get("competitor") or "").strip()
+ csv_text = str(body.get("csvText") or "")
+
+ if not property_id or not competitor or not csv_text.strip():
+ raise HTTPException(
+ status_code=400,
+ detail="propertyId, competitor, and csvText required",
+ )
+
+ try:
+ from website_profiling.integrations.keywords.competitor_csv import ( # type: ignore[import]
+ parse_competitor_keyword_csv,
+ )
+ from website_profiling.integrations.keywords.competitor_gap_store import ( # type: ignore[import]
+ merge_competitor_keyword_import,
+ )
+
+ rows = parse_competitor_keyword_csv(csv_text, competitor=competitor)
+ merged = merge_competitor_keyword_import(conn, property_id, competitor, rows)
+ return {
+ "count": len(rows),
+ "rows": rows[:500],
+ "mergedCount": len(merged),
+ "mergedRows": merged[:500],
+ }
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=f"Competitor keyword import failed: {exc}")
+
+
+# ── POST /api/keywords/content-brief ─────────────────────────────────────────
+
+@router.post("/content-brief")
+def keywords_content_brief(
+ body: dict[str, Any] = Body(default={}),
+) -> dict[str, Any]:
+ keyword = str(body.get("keyword") or "").strip()
+ if not keyword:
+ raise HTTPException(status_code=400, detail="keyword required")
+
+ rows = body.get("rows") or []
+ gaps = body.get("gaps") or []
+
+ try:
+ from website_profiling.llm.content_brief import generate_content_brief # type: ignore[import]
+
+ brief = generate_content_brief(keyword, rows, gaps)
+ return {"brief": brief}
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=f"Content brief generation failed: {exc}")
diff --git a/src/website_profiling/api/routers/logs.py b/src/website_profiling/api/routers/logs.py
new file mode 100644
index 00000000..f8e472db
--- /dev/null
+++ b/src/website_profiling/api/routers/logs.py
@@ -0,0 +1,33 @@
+"""Access log upload and analysis — /api/logs/*."""
+from __future__ import annotations
+
+from typing import Annotated, Any
+
+from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile
+from psycopg import Connection
+
+from ..deps import get_db
+
+router = APIRouter(tags=["logs"])
+
+DbDep = Annotated[Connection, Depends(get_db)]
+
+
+@router.post("/logs/upload")
+def logs_upload(
+ conn: DbDep,
+ propertyId: int = Form(...),
+ file: UploadFile = File(...),
+) -> dict[str, Any]:
+ if not propertyId:
+ raise HTTPException(status_code=400, detail="propertyId required")
+ content = file.file.read().decode("utf-8", errors="replace")
+ try:
+ from website_profiling.tools.log_analysis import parse_and_store_access_log
+
+ result = parse_and_store_access_log(conn, propertyId, content)
+ return result if isinstance(result, dict) else {"ok": True}
+ except ImportError:
+ raise HTTPException(status_code=501, detail="Log analysis module unavailable")
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
diff --git a/src/website_profiling/api/routers/mcp_tools.py b/src/website_profiling/api/routers/mcp_tools.py
new file mode 100644
index 00000000..45cb6b49
--- /dev/null
+++ b/src/website_profiling/api/routers/mcp_tools.py
@@ -0,0 +1,41 @@
+"""MCP audit tool catalog — /api/mcp-tools."""
+from __future__ import annotations
+
+from typing import Any
+
+from fastapi import APIRouter, HTTPException
+
+router = APIRouter(tags=["mcp-tools"])
+
+
+@router.get("/mcp-tools")
+def mcp_tools() -> dict[str, Any]:
+ try:
+ from website_profiling.tools.audit_tools.registry import (
+ TOOL_DEFINITIONS,
+ get_tool_meta,
+ mcp_tool_names,
+ )
+ from website_profiling.tools.audit_tools.tool_domains import (
+ MCP_DOMAIN_BUNDLES,
+ classify_tool_domain,
+ )
+
+ bundle_sets = {b: mcp_tool_names(b) for b in MCP_DOMAIN_BUNDLES.keys()}
+ tools = []
+ for spec in TOOL_DEFINITIONS:
+ name = spec.get("name", "")
+ if not name:
+ continue
+ meta = get_tool_meta(name) or {}
+ domain = meta.get("domain") or classify_tool_domain(name)
+ in_bundles = [b for b, names in bundle_sets.items() if name in names]
+ tools.append({
+ "name": name,
+ "description": spec.get("description", ""),
+ "domain": domain,
+ "bundles": in_bundles,
+ })
+ return {"tools": tools, "bundles": list(MCP_DOMAIN_BUNDLES.keys())}
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
diff --git a/src/website_profiling/api/routers/ollama.py b/src/website_profiling/api/routers/ollama.py
new file mode 100644
index 00000000..315d64ec
--- /dev/null
+++ b/src/website_profiling/api/routers/ollama.py
@@ -0,0 +1,65 @@
+"""Ollama LLM runtime status — /api/ollama/*."""
+from __future__ import annotations
+
+from typing import Annotated, Any
+
+from fastapi import APIRouter, Depends
+from psycopg import Connection
+
+from ..deps import get_db
+
+router = APIRouter(tags=["ollama"])
+
+DbDep = Annotated[Connection, Depends(get_db)]
+
+DEFAULT_BASE = "http://127.0.0.1:11434"
+
+
+@router.get("/ollama/status")
+def ollama_status(conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.config_store import read_llm_config
+ from website_profiling.llm.ollama_catalog import (
+ fetch_ollama_models,
+ model_is_configured,
+ models_support_tools,
+ )
+
+ cfg = read_llm_config(conn)
+ base_url = str(cfg.get("llm_base_url") or DEFAULT_BASE).rstrip("/")
+ configured_model = str(cfg.get("llm_model") or "").strip()
+
+ result = fetch_ollama_models(base_url)
+ if not result.get("ok"):
+ return {
+ "ok": False,
+ "baseUrl": result.get("baseUrl", base_url),
+ "configuredModel": configured_model,
+ "error": result.get("error") or "Cannot reach Ollama. Is it running?",
+ "models": [],
+ "cloudCatalogOk": False,
+ "localOk": False,
+ }
+
+ models = result.get("models") or []
+ model_installed = model_is_configured(models, configured_model)
+ configured_entry = next(
+ (m for m in models if str(m.get("name") or "").lower() == configured_model.lower()),
+ None,
+ )
+
+ return {
+ "ok": True,
+ "baseUrl": result.get("baseUrl", base_url),
+ "configuredModel": configured_model,
+ "modelInstalled": model_installed,
+ "supportsTools": (
+ "tools" in (configured_entry.get("capabilities") or [])
+ if configured_entry
+ else models_support_tools(models)
+ ),
+ "cloudCatalogOk": result.get("cloudCatalogOk", False),
+ "localOk": result.get("localOk", False),
+ "catalogSource": "live",
+ "cloudModelCount": sum(1 for m in models if m.get("source") == "cloud"),
+ "models": models,
+ }
diff --git a/src/website_profiling/api/routers/page_coach.py b/src/website_profiling/api/routers/page_coach.py
new file mode 100644
index 00000000..3567175e
--- /dev/null
+++ b/src/website_profiling/api/routers/page_coach.py
@@ -0,0 +1,48 @@
+"""Internal link page coach — /api/links/page-coach."""
+from __future__ import annotations
+
+from typing import Annotated, Any, Optional
+
+from fastapi import APIRouter, Depends, HTTPException
+from psycopg import Connection
+from pydantic import BaseModel
+
+from ..deps import get_db
+
+router = APIRouter(tags=["page-coach"])
+
+DbDep = Annotated[Connection, Depends(get_db)]
+
+
+class PageCoachBody(BaseModel):
+ url: Optional[str] = None
+ refresh: bool = False
+ currentType: Optional[str] = None
+ currentId: Optional[int] = None
+ baselineType: Optional[str] = None
+ baselineId: Optional[int] = None
+ propertyId: Optional[int] = None
+
+
+@router.post("/links/page-coach")
+def page_coach(body: PageCoachBody, conn: DbDep) -> dict[str, Any]:
+ url = (body.url or "").strip()
+ if not url:
+ raise HTTPException(status_code=400, detail="url required")
+ try:
+ from website_profiling.tools.page_coach import run_page_coach
+
+ return run_page_coach(
+ conn,
+ url=url,
+ refresh=body.refresh,
+ current_type=body.currentType,
+ current_id=body.currentId,
+ baseline_type=body.baselineType,
+ baseline_id=body.baselineId,
+ property_id=body.propertyId,
+ )
+ except ImportError:
+ raise HTTPException(status_code=501, detail="Page coach module unavailable")
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
diff --git a/src/website_profiling/api/routers/page_markdown.py b/src/website_profiling/api/routers/page_markdown.py
new file mode 100644
index 00000000..801b6dbe
--- /dev/null
+++ b/src/website_profiling/api/routers/page_markdown.py
@@ -0,0 +1,157 @@
+"""Page markdown routers — /api/page-markdown/*."""
+from __future__ import annotations
+
+from typing import Annotated, Any, Optional
+
+from fastapi import APIRouter, Body, Depends, HTTPException, Query
+from psycopg import Connection
+
+from ..deps import get_db
+from website_profiling.db.markdown_store import (
+ delete_page_markdown_for_run,
+ list_markdown_crawl_runs,
+ list_page_markdown,
+ read_page_markdown,
+)
+
+router = APIRouter(prefix="/page-markdown", tags=["page-markdown"])
+
+DbDep = Annotated[Connection, Depends(get_db)]
+
+
+@router.get("")
+def list_page_markdown_route(
+ conn: DbDep,
+ crawlRunId: int = Query(...),
+ page: int = Query(1, ge=1),
+ limit: int = Query(25, ge=1, le=100),
+ q: Optional[str] = Query(None),
+) -> dict[str, Any]:
+ if not crawlRunId:
+ raise HTTPException(status_code=400, detail="crawlRunId required")
+
+ page = max(1, page)
+ page_size = min(100, max(1, limit))
+ offset = (page - 1) * page_size
+
+ try:
+ result = list_page_markdown(
+ conn,
+ crawlRunId,
+ limit=page_size,
+ offset=offset,
+ query=(q or "").strip(),
+ )
+ items = []
+ for row in result.get("items") or []:
+ extracted = row.get("extracted_at")
+ items.append({
+ "url": row.get("url"),
+ "title": row.get("title"),
+ "word_count": row.get("word_count"),
+ "strategy": row.get("strategy"),
+ "extracted_at": str(extracted) if extracted else None,
+ })
+ total = int(result.get("total") or 0)
+ return {
+ "items": items,
+ "total": total,
+ "page": page,
+ "pageSize": page_size,
+ "totalPages": max(1, -(-total // page_size)),
+ }
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
+
+
+@router.delete("")
+def delete_page_markdown_route(
+ conn: DbDep,
+ body: dict[str, Any] = Body(default={}),
+) -> dict[str, Any]:
+ crawl_run_id = int(body.get("crawlRunId") or 0)
+ if not crawl_run_id:
+ raise HTTPException(status_code=400, detail="crawlRunId required")
+
+ try:
+ deleted = delete_page_markdown_for_run(conn, crawl_run_id)
+ return {"ok": True, "crawlRunId": crawl_run_id, "deletedRows": deleted}
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
+
+
+@router.get("/content")
+def page_markdown_content_route(
+ conn: DbDep,
+ crawlRunId: int = Query(...),
+ url: str = Query(...),
+) -> dict[str, Any]:
+ if not crawlRunId:
+ raise HTTPException(status_code=400, detail="crawlRunId required")
+ if not url:
+ raise HTTPException(status_code=400, detail="url required")
+
+ try:
+ content = read_page_markdown(conn, crawlRunId, url)
+ if not content:
+ raise HTTPException(status_code=404, detail="Not found")
+ extracted = content.get("extracted_at")
+ return {
+ "content": {
+ "url": content.get("url"),
+ "title": content.get("title"),
+ "markdown": content.get("markdown"),
+ "word_count": content.get("word_count"),
+ "strategy": content.get("strategy"),
+ "source_byte_length": content.get("source_byte_length"),
+ "extracted_at": str(extracted) if extracted else None,
+ }
+ }
+ except HTTPException:
+ raise
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
+
+
+@router.post("/extract")
+def page_markdown_extract(
+ conn: DbDep,
+ body: dict[str, Any] = Body(default={}),
+) -> dict[str, Any]:
+ crawl_run_id = int(body.get("crawlRunId") or 0)
+ if not crawl_run_id:
+ raise HTTPException(status_code=400, detail="crawlRunId required")
+
+ strategy = "full_body" if body.get("strategy") == "full_body" else "main_only"
+ overwrite = body.get("overwrite", True)
+ workers = min(16, max(1, int(body.get("workers") or 4)))
+
+ command = f"page-markdown --crawl-run-id {crawl_run_id} --strategy {strategy} --workers {workers}"
+ if not overwrite:
+ command += " --no-overwrite"
+
+ try:
+ from website_profiling.db.pipeline_jobs import enqueue_job
+ import uuid
+
+ job_id = str(uuid.uuid4())
+ ok = enqueue_job(conn, job_id, "page-markdown", command, None, None)
+ if not ok:
+ raise HTTPException(status_code=400, detail="A pipeline job is already running")
+ return {"jobId": job_id, "crawlRunId": crawl_run_id, "strategy": strategy, "overwrite": overwrite}
+ except HTTPException:
+ raise
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
+
+
+@router.get("/runs")
+def page_markdown_runs_route(
+ conn: DbDep,
+ propertyId: Optional[int] = Query(None),
+) -> dict[str, Any]:
+ try:
+ runs = list_markdown_crawl_runs(conn, propertyId)
+ return {"runs": runs}
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
diff --git a/src/website_profiling/api/routers/pipeline.py b/src/website_profiling/api/routers/pipeline.py
new file mode 100644
index 00000000..bd30d04d
--- /dev/null
+++ b/src/website_profiling/api/routers/pipeline.py
@@ -0,0 +1,258 @@
+"""Pipeline job routers — /api/run, /api/jobs."""
+from __future__ import annotations
+
+import re
+import uuid
+from typing import Annotated, Any, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from psycopg import Connection
+
+from ..deps import get_db
+from ..schemas.pipeline import (
+ ALLOWED_COMMANDS,
+ CancelResponse,
+ JobResponse,
+ JobsListResponse,
+ PauseResponse,
+ ResumeResponse,
+ RunPostBody,
+ RunResponse,
+ coerce_llm_state,
+ coerce_pipeline_state,
+ validate_pipeline_run,
+)
+
+router = APIRouter(tags=["pipeline"])
+
+DbDep = Annotated[Connection, Depends(get_db)]
+
+_PAUSE_RUN_ID_RE = re.compile(r"CRAWL_RUN_ID=(\d+)")
+
+
+def _get_pipeline_jobs_db(conn: Connection):
+ """Late import to avoid circular deps at startup."""
+ from website_profiling.db.pipeline_jobs import (
+ cancel_job_in_db,
+ check_flags,
+ enqueue_job,
+ get_active_job,
+ get_job,
+ list_jobs,
+ reconcile_stale_jobs,
+ set_cancel_flag,
+ set_pause_flag,
+ )
+ return locals()
+
+
+# ── POST /api/run ─────────────────────────────────────────────────────────────
+
+@router.post("/run", response_model=RunResponse)
+def run_pipeline(body: RunPostBody, conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.config_store import (
+ read_pipeline_config,
+ read_llm_config,
+ write_llm_config,
+ write_pipeline_config,
+ )
+ from website_profiling.db.pipeline_jobs import enqueue_job, reconcile_stale_jobs
+ from website_profiling.db.property_store import upsert_property_by_domain
+
+ command = body.command or None
+ command_base = command.split()[0] if command else None
+ if command_base is not None and command_base not in {
+ c for c in ALLOWED_COMMANDS if c is not None and c
+ }:
+ raise HTTPException(status_code=400, detail=f"Invalid command: {command_base}")
+
+ # Resolve state — fall back to saved config if not provided
+ raw_state = body.state
+ unknown_keys = [{"key": u.key, "value": u.value} for u in (body.unknownKeys or [])]
+
+ if not raw_state:
+ try:
+ saved_state, saved_unknown = read_pipeline_config(conn)
+ raw_state = saved_state
+ unknown_keys = saved_unknown
+ except Exception as exc:
+ raise HTTPException(
+ status_code=400,
+ detail=f"Missing state and could not load config: {exc}",
+ )
+
+ if not raw_state:
+ raise HTTPException(status_code=400, detail="Missing state object")
+
+ state = coerce_pipeline_state(raw_state)
+
+ # Filter unknown keys
+ safe_unknown = [
+ u for u in unknown_keys
+ if isinstance(u, dict)
+ and not str(u.get("key", "")).startswith("llm_")
+ and not str(u.get("key", "")).startswith("ml_")
+ ]
+
+ # Resolve property ID from start_url
+ start_url = str(state.get("start_url") or "").strip()
+ property_id: int | None = body.propertyId
+ if start_url:
+ from urllib.parse import urlparse
+ hostname = urlparse(start_url).hostname or ""
+ if hostname:
+ try:
+ from website_profiling.db.property_store import (
+ canonical_domain_from_start_url,
+ upsert_property_by_domain,
+ )
+ domain = canonical_domain_from_start_url(start_url)
+ if domain:
+ property_id = upsert_property_by_domain(
+ conn, domain, domain, start_url
+ )
+ except Exception:
+ pass
+ state["active_property_id"] = str(property_id or "")
+
+ # Validate
+ errors = validate_pipeline_run(state, command)
+ if errors:
+ raise HTTPException(status_code=400, detail=" ".join(errors))
+
+ # Save pipeline config
+ str_state = {k: str(v) for k, v in state.items() if v is not None}
+ try:
+ write_pipeline_config(conn, str_state, safe_unknown)
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=f"Failed to save config: {exc}")
+
+ # Save LLM config if provided
+ if body.llmState and isinstance(body.llmState, dict):
+ llm_coerced = coerce_llm_state(body.llmState)
+ str_llm = {k: str(v) for k, v in llm_coerced.items() if not str(k).endswith("_masked")}
+ try:
+ write_llm_config(conn, str_llm)
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=f"Failed to save LLM config: {exc}")
+
+ # Enqueue job
+ job_id = str(uuid.uuid4())
+ try:
+ ok = enqueue_job(conn, job_id, command_base or "full", command, property_id, None)
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
+
+ if not ok:
+ raise HTTPException(status_code=400, detail="An audit job is already running")
+
+ return {"jobId": job_id}
+
+
+# ── GET /api/jobs ─────────────────────────────────────────────────────────────
+
+@router.get("/jobs", response_model=JobsListResponse)
+def list_pipeline_jobs(
+ conn: DbDep,
+ limit: int = Query(20, ge=1, le=100),
+) -> dict[str, Any]:
+ from website_profiling.db.pipeline_jobs import (
+ get_active_job,
+ list_jobs,
+ reconcile_stale_jobs,
+ )
+
+ reconciled = reconcile_stale_jobs(conn)
+ active = get_active_job(conn)
+ jobs = list_jobs(conn, limit)
+ return {"jobs": jobs, "active": active, "reconciled": reconciled}
+
+
+# ── GET /api/jobs/{id} ────────────────────────────────────────────────────────
+
+@router.get("/jobs/{job_id}")
+def get_pipeline_job(job_id: str, conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.pipeline_jobs import get_job
+
+ job = get_job(conn, job_id)
+ if not job:
+ raise HTTPException(status_code=404, detail="Job not found")
+ return {
+ "status": job["status"],
+ "exitCode": job["exitCode"],
+ "log": job["log"],
+ "error": job.get("error"),
+ "logTruncated": job.get("logTruncated", False),
+ }
+
+
+# ── POST /api/jobs/{id}/cancel ────────────────────────────────────────────────
+
+@router.post("/jobs/{job_id}/cancel", response_model=CancelResponse)
+def cancel_pipeline_job(job_id: str, conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.pipeline_jobs import cancel_job_in_db, get_job, set_cancel_flag
+
+ job = get_job(conn, job_id)
+ if not job:
+ raise HTTPException(status_code=404, detail="Job not found")
+
+ if job["status"] not in ("pending", "running"):
+ raise HTTPException(status_code=409, detail="Job is not running")
+
+ # Set the cancel flag — the worker will pick it up and kill the subprocess.
+ set_cancel_flag(conn, job_id)
+ return {"ok": True, "status": job["status"]}
+
+
+# ── POST /api/jobs/{id}/pause ─────────────────────────────────────────────────
+
+@router.post("/jobs/{job_id}/pause", response_model=PauseResponse)
+def pause_pipeline_job(job_id: str, conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.pipeline_jobs import get_job, set_pause_flag
+
+ job = get_job(conn, job_id)
+ if not job:
+ raise HTTPException(status_code=404, detail="Job not found")
+
+ if job["status"] != "running":
+ raise HTTPException(status_code=409, detail="Job is not running")
+
+ set_pause_flag(conn, job_id)
+ return {"ok": True}
+
+
+# ── POST /api/jobs/{id}/resume ────────────────────────────────────────────────
+
+@router.post("/jobs/{job_id}/resume", response_model=ResumeResponse)
+def resume_pipeline_job(job_id: str, conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.pipeline_jobs import enqueue_job, get_job
+
+ job = get_job(conn, job_id)
+ if not job:
+ raise HTTPException(status_code=404, detail="Job not found")
+
+ if job["status"] != "paused":
+ raise HTTPException(status_code=409, detail="Job is not paused")
+
+ # Extract paused crawl run ID from log
+ log_text = str(job.get("log") or "")
+ m = _PAUSE_RUN_ID_RE.search(log_text)
+ if not m:
+ raise HTTPException(status_code=409, detail="No paused crawl run found for this job")
+
+ paused_run_id = int(m.group(1))
+ resume_command = f"--resume-run-id {paused_run_id}"
+ new_job_id = str(uuid.uuid4())
+
+ ok = enqueue_job(
+ conn,
+ new_job_id,
+ "crawl-resume",
+ resume_command,
+ job.get("propertyId"),
+ None,
+ )
+ if not ok:
+ raise HTTPException(status_code=400, detail="An audit job is already running")
+
+ return {"ok": True, "newJobId": new_job_id}
diff --git a/src/website_profiling/api/routers/portfolio.py b/src/website_profiling/api/routers/portfolio.py
new file mode 100644
index 00000000..ed0c8d76
--- /dev/null
+++ b/src/website_profiling/api/routers/portfolio.py
@@ -0,0 +1,35 @@
+"""Portfolio item deletion — /api/portfolio/*."""
+from __future__ import annotations
+
+from typing import Annotated, Any, Optional
+
+from fastapi import APIRouter, Depends, HTTPException
+from psycopg import Connection
+from pydantic import BaseModel
+
+from ..deps import get_db
+from website_profiling.db import portfolio_store
+
+router = APIRouter(tags=["portfolio"])
+
+DbDep = Annotated[Connection, Depends(get_db)]
+
+
+class DeletePortfolioBody(BaseModel):
+ reportId: Optional[int] = None
+ crawlRunId: Optional[int] = None
+
+
+@router.delete("/portfolio/delete")
+def delete_portfolio_item(body: DeletePortfolioBody, conn: DbDep) -> dict[str, Any]:
+ if body.reportId is None and body.crawlRunId is None:
+ raise HTTPException(status_code=400, detail="reportId or crawlRunId required")
+
+ deleted = portfolio_store.delete_portfolio_item(
+ conn,
+ report_id=body.reportId,
+ crawl_run_id=body.crawlRunId,
+ )
+ if not deleted:
+ raise HTTPException(status_code=404, detail="portfolio item not found")
+ return {"ok": True}
diff --git a/src/website_profiling/api/routers/properties.py b/src/website_profiling/api/routers/properties.py
new file mode 100644
index 00000000..43f68197
--- /dev/null
+++ b/src/website_profiling/api/routers/properties.py
@@ -0,0 +1,324 @@
+"""Properties router — /api/properties/*"""
+from __future__ import annotations
+
+from typing import Annotated, Any, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from pydantic import BaseModel
+from psycopg import Connection
+
+from ..deps import get_db
+
+router = APIRouter(tags=["properties"])
+
+DbDep = Annotated[Connection, Depends(get_db)]
+
+
+class PropertyUpsertBody(BaseModel):
+ name: Optional[str] = None
+ canonical_domain: Optional[str] = None
+ site_url: Optional[str] = None
+
+
+class OpsSettingsBody(BaseModel):
+ scheduleCron: Optional[str] = None
+ alertWebhookUrl: Optional[str] = None
+ alertEmail: Optional[str] = None
+
+
+class PresetBody(BaseModel):
+ preset: Optional[str] = None
+
+
+class GoogleCredentialsPatch(BaseModel):
+ refreshToken: Optional[str] = None
+ authMode: Optional[str] = None
+ gscSiteUrl: Optional[str] = None
+ ga4PropertyId: Optional[str] = None
+ dateRangeDays: Optional[int] = None
+ connectedEmail: Optional[str] = None
+
+
+class GoogleCredentialsPostBody(BaseModel):
+ gscSiteUrl: Optional[str] = None
+ ga4PropertyId: Optional[str] = None
+ dateRangeDays: Optional[int] = None
+ refreshToken: Optional[str] = None
+
+
+@router.get("/properties")
+def list_properties(conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.property_store import list_properties_public
+ return {"properties": list_properties_public(conn)}
+
+
+@router.post("/properties", status_code=201)
+def create_property(body: PropertyUpsertBody, conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.property_store import upsert_property_by_domain
+
+ name = (body.name or "").strip()
+ domain = (body.canonical_domain or "").strip().lower()
+ if not name or not domain:
+ raise HTTPException(status_code=400, detail="name and canonical_domain required")
+
+ site_url = (body.site_url or "").strip() or None
+ prop_id = upsert_property_by_domain(conn, name, domain, site_url)
+ return {"id": prop_id, "name": name, "canonical_domain": domain}
+
+
+@router.get("/properties/resolve")
+def resolve_property(
+ conn: DbDep,
+ startUrl: str = Query(..., description="Start URL to resolve a property from"),
+) -> dict[str, Any]:
+ from website_profiling.db.property_store import (
+ canonical_domain_from_start_url,
+ get_property_by_domain,
+ resolve_property_id_from_start_url,
+ )
+
+ start_url = startUrl.strip()
+ if not start_url:
+ raise HTTPException(status_code=400, detail="startUrl required")
+
+ prop_id = resolve_property_id_from_start_url(conn, start_url)
+ domain = canonical_domain_from_start_url(start_url)
+ prop = get_property_by_domain(conn, domain) if domain else None
+ return {
+ "id": prop_id,
+ "canonical_domain": domain,
+ "default_crawl_preset": prop.get("default_crawl_preset") if prop else None,
+ }
+
+
+@router.get("/properties/{property_id}")
+def get_property(property_id: int, conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.property_store import get_property_by_id
+
+ prop = get_property_by_id(conn, property_id)
+ if not prop:
+ raise HTTPException(status_code=404, detail="Property not found")
+ return prop
+
+
+@router.delete("/properties/{property_id}")
+def delete_property_route(property_id: int, conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.property_store import delete_property
+
+ if not delete_property(conn, property_id):
+ raise HTTPException(status_code=404, detail="Property not found")
+ return {"ok": True}
+
+
+@router.get("/properties/{property_id}/ops")
+def get_property_ops_route(property_id: int, conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.property_store import get_property_ops
+
+ ops = get_property_ops(conn, property_id)
+ if not ops:
+ raise HTTPException(status_code=404, detail="Property not found")
+ return ops
+
+
+@router.put("/properties/{property_id}/ops")
+def update_property_ops_route(property_id: int, body: OpsSettingsBody, conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.property_store import get_property_by_id, update_property_ops
+
+ if not get_property_by_id(conn, property_id):
+ raise HTTPException(status_code=404, detail="Property not found")
+
+ update_property_ops(
+ conn,
+ property_id,
+ schedule_cron=body.scheduleCron,
+ alert_webhook_url=body.alertWebhookUrl,
+ alert_email=body.alertEmail,
+ )
+ return {"ok": True}
+
+
+@router.get("/properties/{property_id}/preset")
+def get_property_preset(property_id: int, conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.property_store import get_property_by_id
+
+ prop = get_property_by_id(conn, property_id)
+ if not prop:
+ raise HTTPException(status_code=404, detail="Property not found")
+ return {"default_crawl_preset": prop.get("default_crawl_preset")}
+
+
+@router.put("/properties/{property_id}/preset")
+def update_property_preset(property_id: int, body: PresetBody, conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.property_store import get_property_by_id, update_property_crawl_preset
+
+ if not get_property_by_id(conn, property_id):
+ raise HTTPException(status_code=404, detail="Property not found")
+
+ preset = (body.preset or "").strip() or None
+ update_property_crawl_preset(conn, property_id, preset)
+ return {"ok": True, "default_crawl_preset": preset}
+
+
+@router.post("/properties/{property_id}/authorize")
+def authorize_property_crawl_route(property_id: int, conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.property_store import authorize_property_crawl, get_property_by_id
+
+ if not get_property_by_id(conn, property_id):
+ raise HTTPException(status_code=404, detail="Property not found")
+ authorize_property_crawl(conn, property_id)
+ return {"ok": True}
+
+
+@router.get("/properties/{property_id}/google/status")
+def property_google_status(property_id: int, conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.property_store import get_property_google_status
+
+ status = get_property_google_status(conn, property_id)
+ if not status:
+ raise HTTPException(status_code=404, detail="Property not found")
+ return status
+
+
+@router.post("/properties/{property_id}/google/test")
+def property_google_test(property_id: int, conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.property_store import get_property_by_id
+
+ if not get_property_by_id(conn, property_id):
+ raise HTTPException(status_code=404, detail="Property not found")
+ try:
+ from website_profiling.integrations.google.test import test_google_connection
+ result = test_google_connection(conn, property_id)
+ return result if isinstance(result, dict) else {"ok": True, "log": str(result)}
+ except ImportError:
+ raise HTTPException(status_code=501, detail="Google test unavailable")
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
+
+
+@router.get("/properties/{property_id}/google/properties")
+def property_google_properties(property_id: int, conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.property_store import get_property_by_id
+
+ if not get_property_by_id(conn, property_id):
+ raise HTTPException(status_code=404, detail="Property not found")
+ try:
+ from website_profiling.integrations.google.discover import list_google_properties
+ result = list_google_properties(conn, property_id)
+ return result if isinstance(result, dict) else {"properties": result}
+ except ImportError:
+ raise HTTPException(status_code=501, detail="Google properties discovery unavailable")
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
+
+
+@router.get("/properties/{property_id}/google/links/status")
+def property_google_links_status(property_id: int, conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.property_store import get_property_by_id
+
+ if not get_property_by_id(conn, property_id):
+ raise HTTPException(status_code=404, detail="Property not found")
+ try:
+ from website_profiling.integrations.google.gsc_links_store import read_gsc_links_status
+ return read_gsc_links_status(conn, property_id)
+ except Exception:
+ return {"hasData": False}
+
+
+@router.post("/properties/{property_id}/google/links/import")
+def property_google_links_import(property_id: int, conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.property_store import get_property_by_id
+
+ if not get_property_by_id(conn, property_id):
+ raise HTTPException(status_code=404, detail="Property not found")
+ try:
+ from website_profiling.integrations.google.links import import_gsc_links
+ result = import_gsc_links(conn, property_id)
+ return result if isinstance(result, dict) else {"ok": True, "imported": result}
+ except ImportError:
+ raise HTTPException(status_code=501, detail="GSC links import unavailable")
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
+
+
+def _apply_google_credentials_from_patch(
+ conn: Connection,
+ property_id: int,
+ body: GoogleCredentialsPatch,
+) -> None:
+ from website_profiling.db.property_store import apply_property_google_credentials_patch
+
+ fields_set: set[str] = set()
+ if body.gscSiteUrl is not None:
+ fields_set.add("gsc_site_url")
+ if body.ga4PropertyId is not None:
+ fields_set.add("ga4_property_id")
+ if body.dateRangeDays is not None:
+ fields_set.add("date_range_days")
+ if body.authMode is not None:
+ fields_set.add("auth_mode")
+ if body.connectedEmail is not None:
+ fields_set.add("connected_email")
+ if body.refreshToken is not None:
+ fields_set.add("refresh_token")
+
+ try:
+ apply_property_google_credentials_patch(
+ conn,
+ property_id,
+ refresh_token=body.refreshToken,
+ auth_mode=body.authMode,
+ gsc_site_url=body.gscSiteUrl,
+ ga4_property_id=body.ga4PropertyId,
+ date_range_days=body.dateRangeDays,
+ connected_email=body.connectedEmail,
+ fields_set=frozenset(fields_set) if fields_set else None,
+ )
+ except ValueError as exc:
+ raise HTTPException(status_code=400, detail=str(exc)) from exc
+
+
+@router.patch("/properties/{property_id}/google/credentials")
+def patch_property_google_credentials(
+ property_id: int, body: GoogleCredentialsPatch, conn: DbDep
+) -> dict[str, Any]:
+ from website_profiling.db.property_store import get_property_by_id
+
+ if not get_property_by_id(conn, property_id):
+ raise HTTPException(status_code=404, detail="Property not found")
+ _apply_google_credentials_from_patch(conn, property_id, body)
+ return {"ok": True}
+
+
+@router.post("/properties/{property_id}/google/credentials")
+def post_property_google_credentials(
+ property_id: int, body: GoogleCredentialsPostBody, conn: DbDep
+) -> dict[str, Any]:
+ from website_profiling.db.property_store import get_property_by_id, get_property_google_public_status
+
+ if not get_property_by_id(conn, property_id):
+ raise HTTPException(status_code=404, detail="Property not found")
+
+ patch = GoogleCredentialsPatch()
+ fields_set = body.model_fields_set
+ if "gscSiteUrl" in fields_set:
+ patch.gscSiteUrl = body.gscSiteUrl
+ if "ga4PropertyId" in fields_set:
+ patch.ga4PropertyId = body.ga4PropertyId
+ if "dateRangeDays" in fields_set and body.dateRangeDays is not None:
+ patch.dateRangeDays = body.dateRangeDays
+ if isinstance(body.refreshToken, str) and body.refreshToken.strip():
+ patch.refreshToken = body.refreshToken.strip()
+ patch.authMode = "oauth"
+
+ _apply_google_credentials_from_patch(conn, property_id, patch)
+ return {"ok": True, "status": get_property_google_public_status(conn, property_id)}
+
+
+@router.post("/properties/{property_id}/google/disconnect")
+def post_property_google_disconnect(property_id: int, conn: DbDep) -> dict[str, Any]:
+ from website_profiling.db.property_store import disconnect_property_google, get_property_by_id
+
+ if not get_property_by_id(conn, property_id):
+ raise HTTPException(status_code=404, detail="Property not found")
+ disconnect_property_google(conn, property_id)
+ return {"ok": True}
diff --git a/src/website_profiling/api/routers/report.py b/src/website_profiling/api/routers/report.py
new file mode 100644
index 00000000..507c6aaf
--- /dev/null
+++ b/src/website_profiling/api/routers/report.py
@@ -0,0 +1,83 @@
+"""Report data routers — /api/report/*."""
+from __future__ import annotations
+
+from typing import Annotated, Any, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from psycopg import Connection
+
+from ..deps import get_db
+from ..services.report_loader import (
+ SECTION_KEYS,
+ get_crawl_preview_payload,
+ get_mobile_desktop_delta,
+ get_report_payload,
+ list_audit_history,
+ list_crawl_runs,
+ list_reports,
+)
+
+router = APIRouter(prefix="/report", tags=["report"])
+
+DbDep = Annotated[Connection, Depends(get_db)]
+
+
+@router.get("/meta")
+def report_meta(conn: DbDep) -> dict[str, Any]:
+ return {
+ "reports": list_reports(conn),
+ "crawlRuns": list_crawl_runs(conn),
+ }
+
+
+@router.get("/payload")
+def report_payload(
+ conn: DbDep,
+ reportId: Optional[int] = Query(None),
+ domain: Optional[str] = Query(None),
+ section: Optional[str] = Query(None),
+) -> dict[str, Any]:
+ if section is not None and section not in SECTION_KEYS:
+ raise HTTPException(status_code=400, detail="Invalid section")
+ payload = get_report_payload(conn, reportId, domain, section)
+ if payload is None:
+ raise HTTPException(status_code=404, detail="Report not found")
+ if section:
+ return {"payload": payload, "section": section}
+ return {"payload": payload}
+
+
+@router.get("/history")
+def report_history(
+ conn: DbDep,
+ propertyId: Optional[int] = Query(None),
+ domain: Optional[str] = Query(None),
+ limit: int = Query(20, ge=1, le=100),
+) -> dict[str, Any]:
+ history = list_audit_history(conn, propertyId, domain, limit)
+ return {"history": history}
+
+
+@router.get("/crawl-payload")
+def crawl_payload(
+ conn: DbDep,
+ crawlRunId: Optional[int] = Query(None),
+) -> dict[str, Any]:
+ if not crawlRunId or crawlRunId <= 0:
+ raise HTTPException(status_code=400, detail="Invalid crawlRunId")
+ try:
+ payload = get_crawl_preview_payload(conn, crawlRunId)
+ except ValueError as e:
+ raise HTTPException(status_code=404, detail=str(e))
+ return {"payload": payload}
+
+
+@router.get("/mobile-delta")
+def mobile_delta(
+ conn: DbDep,
+ id: Optional[int] = Query(None),
+) -> dict[str, Any]:
+ if not id:
+ raise HTTPException(status_code=400, detail="id required")
+ deltas = get_mobile_desktop_delta(conn, id)
+ return {"deltas": deltas}
diff --git a/src/website_profiling/api/routers/report_audit_tool.py b/src/website_profiling/api/routers/report_audit_tool.py
new file mode 100644
index 00000000..53a11ab0
--- /dev/null
+++ b/src/website_profiling/api/routers/report_audit_tool.py
@@ -0,0 +1,40 @@
+"""Audit tool dispatch — POST /api/report/audit-tool."""
+from __future__ import annotations
+
+from typing import Annotated, Any, Optional
+
+from fastapi import APIRouter, Depends, HTTPException
+from psycopg import Connection
+from pydantic import BaseModel
+
+from ..deps import get_db
+
+router = APIRouter(prefix="/report", tags=["report-audit-tool"])
+
+DbDep = Annotated[Connection, Depends(get_db)]
+
+
+class AuditToolBody(BaseModel):
+ toolName: str
+ propertyId: int
+ reportId: Optional[int] = None
+ args: dict[str, Any] = {}
+
+
+@router.post("/audit-tool")
+def run_audit_tool(body: AuditToolBody, conn: DbDep) -> dict[str, Any]:
+ if not body.toolName or not body.propertyId:
+ raise HTTPException(status_code=400, detail="toolName and propertyId required")
+
+ try:
+ from website_profiling.tools.audit_tools import AuditToolContext
+ from website_profiling.tools.audit_tools.registry import dispatch_tool
+
+ context = AuditToolContext(
+ property_id=body.propertyId,
+ report_id=body.reportId,
+ )
+ result = dispatch_tool(body.toolName, body.args, context=context, conn=conn)
+ return {"result": result}
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
diff --git a/src/website_profiling/api/routers/report_export.py b/src/website_profiling/api/routers/report_export.py
new file mode 100644
index 00000000..024548a0
--- /dev/null
+++ b/src/website_profiling/api/routers/report_export.py
@@ -0,0 +1,72 @@
+"""Report export downloads — /api/report/export*."""
+from __future__ import annotations
+
+from typing import Annotated, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from fastapi.responses import Response
+from psycopg import Connection
+
+from ..deps import get_db
+
+router = APIRouter(prefix="/report", tags=["report-export"])
+
+DbDep = Annotated[Connection, Depends(get_db)]
+
+EXPORT_FORMATS = {"csv", "json"}
+
+
+@router.get("/export")
+def export_report(
+ conn: DbDep,
+ format: str = Query("csv"),
+ reportId: Optional[int] = Query(None),
+) -> Response:
+ if format not in EXPORT_FORMATS:
+ raise HTTPException(status_code=400, detail=f"Invalid format. Use one of {sorted(EXPORT_FORMATS)}")
+
+ try:
+ if format == "csv":
+ from website_profiling.tools.export_audit import export_audit_csv as _export
+ content = _export(conn, reportId)
+ return Response(
+ content=content if isinstance(content, bytes) else content.encode(),
+ media_type="text/csv",
+ headers={"Content-Disposition": "attachment; filename=report.csv"},
+ )
+ if format == "json":
+ import json
+ from website_profiling.tools.export_audit import export_audit_json as _export
+ content = _export(conn, reportId)
+ body = json.dumps(content) if not isinstance(content, (str, bytes)) else content
+ return Response(
+ content=body if isinstance(body, bytes) else body.encode(),
+ media_type="application/json",
+ headers={"Content-Disposition": "attachment; filename=report.json"},
+ )
+ except ImportError as exc:
+ raise HTTPException(status_code=501, detail=f"Export module unavailable: {exc}")
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
+
+ raise HTTPException(status_code=500, detail="Export failed")
+
+
+@router.get("/export-sitemap")
+def export_sitemap(
+ conn: DbDep,
+ reportId: Optional[int] = Query(None),
+) -> Response:
+ try:
+ from website_profiling.tools.export_sitemap import export_sitemap as _export
+ content = _export(conn, reportId)
+ return Response(
+ content=content if isinstance(content, bytes) else content.encode(),
+ media_type="application/xml",
+ headers={"Content-Disposition": "attachment; filename=sitemap.xml"},
+ )
+ except ImportError:
+ raise HTTPException(status_code=501, detail="Sitemap export unavailable")
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
+
diff --git a/src/website_profiling/api/routers/report_portfolio.py b/src/website_profiling/api/routers/report_portfolio.py
new file mode 100644
index 00000000..28dac5a0
--- /dev/null
+++ b/src/website_profiling/api/routers/report_portfolio.py
@@ -0,0 +1,54 @@
+"""Portfolio report widget — GET /api/report/portfolio."""
+from __future__ import annotations
+
+from typing import Annotated, Any, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from psycopg import Connection
+
+from ..deps import get_db
+from ..services.portfolio_loader import get_portfolio_response
+
+router = APIRouter(prefix="/report", tags=["report-portfolio"])
+
+DbDep = Annotated[Connection, Depends(get_db)]
+
+
+@router.get("/portfolio")
+def report_portfolio(
+ conn: DbDep,
+ widget: str = Query("full"),
+ ids: Optional[str] = Query(None),
+ reportId: Optional[int] = Query(None),
+ crawlRunId: Optional[int] = Query(None),
+) -> dict[str, Any]:
+ """Return portfolio data — groups, crawl history, summary, or single card."""
+ valid_widgets = {"full", "groups", "summary", "card"}
+ if widget not in valid_widgets:
+ raise HTTPException(status_code=400, detail="Invalid widget")
+
+ if widget == "card" and reportId is None and crawlRunId is None:
+ raise HTTPException(
+ status_code=400, detail="reportId or crawlRunId required for card widget"
+ )
+
+ id_list: list[int] = []
+ if ids:
+ for s in ids.split(","):
+ try:
+ n = int(s.strip())
+ if n > 0:
+ id_list.append(n)
+ except ValueError:
+ pass
+
+ try:
+ return get_portfolio_response(
+ conn,
+ widget=widget,
+ ids=id_list,
+ report_id=reportId,
+ crawl_run_id=crawlRunId,
+ )
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
diff --git a/src/website_profiling/api/routers/schedule.py b/src/website_profiling/api/routers/schedule.py
new file mode 100644
index 00000000..fef4b7c2
--- /dev/null
+++ b/src/website_profiling/api/routers/schedule.py
@@ -0,0 +1,22 @@
+"""Scheduled crawl checks — /api/schedule/*."""
+from __future__ import annotations
+
+from typing import Any
+
+from fastapi import APIRouter, HTTPException
+
+router = APIRouter(tags=["schedule"])
+
+
+@router.post("/schedule/check")
+def schedule_check() -> dict[str, Any]:
+ try:
+ from website_profiling.tools import schedule_runner
+
+ result = schedule_runner.run()
+ return result if isinstance(result, dict) else {"ok": True}
+ except ImportError:
+ pass
+ except Exception as exc:
+ raise HTTPException(status_code=500, detail=str(exc))
+ return {"ok": True}
diff --git a/src/website_profiling/api/schemas/__init__.py b/src/website_profiling/api/schemas/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/website_profiling/api/schemas/chat.py b/src/website_profiling/api/schemas/chat.py
new file mode 100644
index 00000000..ca4b2d8b
--- /dev/null
+++ b/src/website_profiling/api/schemas/chat.py
@@ -0,0 +1,41 @@
+"""Chat request/response Pydantic schemas."""
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from pydantic import BaseModel
+
+
+class ChatRequest(BaseModel):
+ sessionId: int
+ propertyId: int
+ message: str
+ reportId: Optional[int] = None
+
+
+class ChatSessionCreate(BaseModel):
+ propertyId: int
+ title: str = "New chat"
+
+
+class ChatSessionResponse(BaseModel):
+ id: int
+ propertyId: int
+ title: str
+ createdAt: str
+ updatedAt: str
+
+
+class ChatMessageResponse(BaseModel):
+ id: int
+ role: str
+ content: str
+ tool_name: Optional[str] = None
+ tool_args: Optional[dict[str, Any]] = None
+ tool_result: Optional[dict[str, Any]] = None
+ created_at: str
+
+
+class ArtifactUpdateBody(BaseModel):
+ title: Optional[str] = None
+ pinned: Optional[bool] = None
diff --git a/src/website_profiling/api/schemas/pipeline.py b/src/website_profiling/api/schemas/pipeline.py
new file mode 100644
index 00000000..6580def9
--- /dev/null
+++ b/src/website_profiling/api/schemas/pipeline.py
@@ -0,0 +1,155 @@
+"""Pipeline job and config Pydantic schemas."""
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field
+
+# ── Config field type registry (mirrors pipelineConfigSchema.ts) ─────────────
+
+# bool fields — coerce to Python bool
+_BOOL_KEYS: frozenset[str] = frozenset({
+ "run_crawl", "run_report", "run_keywords", "run_lighthouse", "run_plot",
+ "run_security", "run_enrich", "run_google", "run_page_markdown",
+ "ignore_robots", "allow_external", "store_outlinks", "store_content_excerpt",
+ "store_page_html", "run_content_analysis", "probe_image_inventory",
+ "compare_mobile_desktop", "lighthouse_run_mobile", "enable_ner",
+ "enable_rich_results_validation", "ner_only_top_pages",
+ "enable_hreflang_validation", "enable_crux_summary",
+ "enable_executive_summary", "enable_google_keyword_planner",
+ "enable_competitor_keywords", "export_csv", "export_json", "export_html",
+ "export_pdf", "enable_bing_backlinks",
+})
+
+# tristate fields — 'auto' | 'true' | 'false'
+_TRISTATE_KEYS: frozenset[str] = frozenset({
+ "crawl_render_mode_tristate",
+})
+
+# Keys written internally by the server (not shown in UI)
+INTERNAL_PIPELINE_KEYS: frozenset[str] = frozenset({"active_property_id"})
+
+ALLOWED_COMMANDS: frozenset[str | None] = frozenset({
+ None, "", "crawl", "report", "plot", "lighthouse", "keywords",
+ "keywords --enrich-google", "warnings", "enrich", "google", "page-markdown",
+})
+
+
+def coerce_pipeline_state(raw: dict[str, Any]) -> dict[str, Any]:
+ """Coerce raw state values to correct Python types, mirroring run/route.ts logic."""
+ out: dict[str, Any] = {}
+ for key, val in raw.items():
+ if key.startswith("llm_"):
+ continue
+ if key in _BOOL_KEYS:
+ out[key] = val is True or val == "true"
+ elif key in _TRISTATE_KEYS:
+ s = str(val or "auto").lower()
+ out[key] = "true" if s == "true" else "false" if s == "false" else "auto"
+ else:
+ out[key] = "" if val is None else str(val)
+ return out
+
+
+def coerce_llm_state(raw: dict[str, Any]) -> dict[str, Any]:
+ """Coerce LLM config state, mirroring run/route.ts llm coercion."""
+ # LLM fields that are booleans
+ _LLM_BOOL_KEYS = frozenset({
+ "llm_chat_unlimited_tool_rounds",
+ "llm_reasoning_enabled",
+ })
+ out: dict[str, Any] = {}
+ for key, val in raw.items():
+ if key.endswith("_masked"):
+ continue
+ if key in _LLM_BOOL_KEYS:
+ out[key] = val is True or val == "true"
+ else:
+ out[key] = "" if val is None else str(val)
+ # preserve _masked flags
+ if raw.get(f"{key}_masked") is True:
+ out[f"{key}_masked"] = True
+ return out
+
+
+def validate_pipeline_run(state: dict[str, Any], command: str | None) -> list[str]:
+ """Return validation error messages (empty list = OK)."""
+ errors: list[str] = []
+ start_url = str(state.get("start_url") or "").strip()
+
+ def needs_start_url() -> bool:
+ if command == "crawl":
+ return True
+ if command in ("report", "keywords"):
+ return True
+ if command is None:
+ run_crawl = state.get("run_crawl", True)
+ run_report = state.get("run_report", True)
+ if isinstance(run_crawl, str):
+ run_crawl = run_crawl.lower() == "true"
+ if isinstance(run_report, str):
+ run_report = run_report.lower() == "true"
+ return bool(run_crawl) or bool(run_report)
+ return False
+
+ if needs_start_url() and not start_url:
+ errors.append("Site URL is required. Enter it in Audit settings before continuing.")
+ return errors
+
+
+# ── Request / response models ─────────────────────────────────────────────────
+
+class UnknownKeyEntry(BaseModel):
+ key: str
+ value: str
+
+
+class RunPostBody(BaseModel):
+ command: Optional[str] = None
+ state: Optional[dict[str, Any]] = None
+ unknownKeys: list[UnknownKeyEntry] = Field(default_factory=list)
+ llmState: Optional[dict[str, Any]] = None
+ propertyId: Optional[int] = None
+ python: Optional[str] = None
+ repoRoot: Optional[str] = None
+
+
+class RunResponse(BaseModel):
+ jobId: str
+
+
+class JobResponse(BaseModel):
+ id: str
+ jobType: str
+ status: str
+ exitCode: Optional[int] = None
+ log: str = ""
+ error: Optional[str] = None
+ logTruncated: bool = False
+ propertyId: Optional[int] = None
+ startedAt: Optional[str] = None
+ finishedAt: Optional[str] = None
+ command: Optional[str] = None
+
+
+class JobsListResponse(BaseModel):
+ jobs: list[dict[str, Any]]
+ active: Optional[dict[str, Any]] = None
+ reconciled: int = 0
+
+
+class CancelResponse(BaseModel):
+ ok: bool
+ status: str
+ error: Optional[str] = None
+
+
+class PauseResponse(BaseModel):
+ ok: bool
+ error: Optional[str] = None
+
+
+class ResumeResponse(BaseModel):
+ ok: bool
+ newJobId: Optional[str] = None
+ error: Optional[str] = None
diff --git a/src/website_profiling/api/services/__init__.py b/src/website_profiling/api/services/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/website_profiling/api/services/portfolio_loader.py b/src/website_profiling/api/services/portfolio_loader.py
new file mode 100644
index 00000000..c449c487
--- /dev/null
+++ b/src/website_profiling/api/services/portfolio_loader.py
@@ -0,0 +1,606 @@
+"""Portfolio grouping for /api/report/portfolio — port of web/src/lib/homePortfolio.ts."""
+from __future__ import annotations
+
+import re
+from datetime import datetime
+from typing import Any, Callable, Optional
+from urllib.parse import urlparse
+
+from psycopg import Connection
+
+from website_profiling.db.report_store import read_report_payload
+
+from .report_loader import (
+ list_crawl_run_summaries,
+ list_crawl_runs,
+ list_reports,
+ slice_payload_for_section,
+)
+
+PORTFOLIO_CATEGORY_ORDER = (
+ "technical_seo",
+ "performance",
+ "core_web_vitals",
+ "link_health",
+ "security",
+ "html_accessibility",
+ "mobile",
+ "intelligence",
+)
+
+EMPTY_ISSUE_COUNTS = {"critical": 0, "high": 0, "medium": 0, "low": 0}
+
+DATA_SOURCE_IDS = frozenset({
+ "crawl",
+ "lighthouse",
+ "search_console",
+ "analytics",
+ "backlinks",
+})
+
+UNKNOWN_BRAND = "Unknown property"
+EM_DASH = "—"
+
+
+def _extract_hostname(url: str | None) -> str:
+ if not url:
+ return ""
+ try:
+ host = urlparse(str(url)).hostname
+ return host.lower() if host else ""
+ except Exception:
+ return ""
+
+
+def _slugify_domain(name: str | None) -> str:
+ if not name:
+ return ""
+ s = re.sub(r"[^a-z0-9]+", "-", str(name).strip().lower()).strip("-")
+ return s
+
+
+def _canonical_domain_from_payload(
+ payload: dict[str, Any],
+ start_url_by_run_id: dict[int, str],
+) -> str:
+ run_id = payload.get("crawl_run_id")
+ run_id = int(run_id) if run_id is not None else None
+ run_start = start_url_by_run_id.get(run_id, "") if run_id is not None else ""
+ top_pages = payload.get("top_pages") or []
+ links = payload.get("links") or []
+ fallback = ""
+ if top_pages and isinstance(top_pages[0], dict):
+ fallback = str(top_pages[0].get("url") or "")
+ if not fallback and links and isinstance(links[0], dict):
+ fallback = str(links[0].get("url") or "")
+ start_domain = _extract_hostname(run_start)
+ fallback_domain = _extract_hostname(fallback)
+ return (start_domain or fallback_domain or "").lower()
+
+
+def _crawled_url_count(payload: dict[str, Any]) -> int:
+ scope = (payload.get("report_meta") or {}).get("crawl_scope") or {}
+ pages = scope.get("pages_crawled")
+ if pages is not None:
+ try:
+ n = int(pages)
+ if n > 0:
+ return n
+ except (TypeError, ValueError):
+ pass
+ summary = payload.get("summary") or {}
+ total = summary.get("total_urls")
+ if total is not None:
+ try:
+ n = int(total)
+ if n > 0:
+ return n
+ except (TypeError, ValueError):
+ pass
+ links = payload.get("links") or []
+ return len(links) if links else 0
+
+
+def _score_from_categories(categories: list[dict[str, Any]]) -> int | None:
+ nums = [
+ float(c["score"])
+ for c in categories
+ if isinstance(c.get("score"), (int, float))
+ ]
+ if not nums:
+ return None
+ return round(sum(nums) / len(nums))
+
+
+def _issue_counts_from_payload(payload: dict[str, Any]) -> tuple[dict[str, int], int]:
+ counts = dict(EMPTY_ISSUE_COUNTS)
+ for cat in payload.get("categories") or []:
+ for iss in cat.get("issues") or []:
+ p = str(iss.get("priority") or "Medium")
+ if p == "Critical":
+ counts["critical"] += 1
+ elif p == "High":
+ counts["high"] += 1
+ elif p == "Low":
+ counts["low"] += 1
+ else:
+ counts["medium"] += 1
+ total = sum(counts.values())
+ return counts, total
+
+
+def _category_score(payload: dict[str, Any], cat_id: str) -> int | None:
+ for cat in payload.get("categories") or []:
+ if cat.get("id") == cat_id and isinstance(cat.get("score"), (int, float)):
+ return round(float(cat["score"]))
+ return None
+
+
+def _lh_scores(payload: dict[str, Any]) -> tuple[int | None, int | None]:
+ summary = payload.get("lighthouse_summary")
+ if not isinstance(summary, dict):
+ return None, None
+ mm = summary.get("median_metrics") or {}
+ cs = summary.get("category_scores") or {}
+ perf_raw = mm.get("performance_score") or cs.get("performance")
+ seo_raw = mm.get("seo_score") or cs.get("seo")
+ perf = round(float(perf_raw)) if isinstance(perf_raw, (int, float)) else None
+ seo = round(float(seo_raw)) if isinstance(seo_raw, (int, float)) else None
+ return perf, seo
+
+
+def _category_snapshots(payload: dict[str, Any]) -> list[dict[str, Any]]:
+ cats = payload.get("categories") or []
+ by_id = {str(c.get("id") or ""): c for c in cats}
+ out: list[dict[str, Any]] = []
+
+ def push(cat_id: str) -> None:
+ cat = by_id.get(cat_id)
+ if not cat or not isinstance(cat.get("score"), (int, float)):
+ return
+ out.append({
+ "id": cat_id,
+ "name": str(cat.get("name") or cat_id),
+ "score": round(float(cat["score"])),
+ "issueCount": len(cat.get("issues") or []),
+ })
+
+ for cat_id in PORTFOLIO_CATEGORY_ORDER:
+ push(cat_id)
+ for cat in cats:
+ cat_id = str(cat.get("id") or "")
+ if not cat_id or any(r["id"] == cat_id for r in out):
+ continue
+ if not isinstance(cat.get("score"), (int, float)):
+ continue
+ out.append({
+ "id": cat_id,
+ "name": str(cat.get("name") or cat_id),
+ "score": round(float(cat["score"])),
+ "issueCount": len(cat.get("issues") or []),
+ })
+ return out
+
+
+def _seo_signals(payload: dict[str, Any]) -> dict[str, int] | None:
+ s = payload.get("seo_health")
+ if not isinstance(s, dict):
+ return None
+ return {
+ "missingTitles": int(s.get("missing_title") or 0),
+ "missingMetaDesc": int(s.get("missing_meta_desc") or 0),
+ "thinContent": int(s.get("thin_content") or 0),
+ "h1Issues": int(s.get("h1_zero") or 0) + int(s.get("h1_multi") or 0),
+ }
+
+
+def _median_word_count(payload: dict[str, Any]) -> int | None:
+ median = (payload.get("content_analytics") or {}).get("word_count_stats", {}).get("median")
+ return round(float(median)) if isinstance(median, (int, float)) else None
+
+
+def _median_response_ms(payload: dict[str, Any]) -> int | None:
+ median = (payload.get("response_time_stats") or {}).get("p50")
+ return round(float(median)) if isinstance(median, (int, float)) else None
+
+
+def _data_sources(payload: dict[str, Any]) -> list[str] | None:
+ raw = (payload.get("report_meta") or {}).get("data_sources") or []
+ out = [str(s) for s in raw if str(s) in DATA_SOURCE_IDS]
+ return out or None
+
+
+def _crawl_config_from_payload(
+ payload: dict[str, Any],
+ run_meta: dict[str, Any] | None,
+) -> dict[str, Any] | None:
+ scope = (payload.get("report_meta") or {}).get("crawl_scope")
+ if not scope and not (run_meta or {}).get("render_mode") and not (run_meta or {}).get("discovery_mode"):
+ return None
+ cfg: dict[str, Any] = dict(scope) if isinstance(scope, dict) else {}
+ if run_meta:
+ if run_meta.get("render_mode") and "render_mode" not in cfg:
+ cfg["render_mode"] = run_meta["render_mode"]
+ if run_meta.get("discovery_mode"):
+ cfg["discovery_mode"] = run_meta["discovery_mode"]
+ return cfg or None
+
+
+def _crawl_config_from_summary(row: dict[str, Any]) -> dict[str, Any] | None:
+ if not row.get("render_mode") and not row.get("discovery_mode") and not row.get("url_count"):
+ return None
+ return {
+ "pages_crawled": row.get("url_count"),
+ "render_mode": row.get("render_mode"),
+ "discovery_mode": row.get("discovery_mode"),
+ }
+
+
+def _to_display_datetime(value: str | None) -> str:
+ if not value:
+ return ""
+ try:
+ if isinstance(value, datetime):
+ return value.isoformat()
+ dt = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+ return dt.isoformat()
+ except Exception:
+ return str(value)
+
+
+def _generated_at_ms(value: str | None) -> float:
+ if not value:
+ return 0.0
+ try:
+ return datetime.fromisoformat(str(value).replace("Z", "+00:00")).timestamp() * 1000
+ except Exception:
+ return 0.0
+
+
+def _title_coverage_pct(with_title: int, url_count: int) -> int:
+ if url_count <= 0:
+ return 0
+ return round((with_title / url_count) * 100)
+
+
+def load_portfolio_maps(conn: Connection) -> dict[str, Any]:
+ crawl_rows = list_crawl_runs(conn)
+ start_url_by_run_id = {int(r["id"]): r["start_url"] for r in crawl_rows}
+ run_created_at_by_run_id = {int(r["id"]): r["created_at"] for r in crawl_rows}
+ run_meta_by_run_id = {
+ int(r["id"]): {
+ "render_mode": r.get("render_mode"),
+ "discovery_mode": r.get("discovery_mode"),
+ }
+ for r in crawl_rows
+ }
+ crawl_summaries = list_crawl_run_summaries(conn)
+ return {
+ "start_url_by_run_id": start_url_by_run_id,
+ "run_created_at_by_run_id": run_created_at_by_run_id,
+ "run_meta_by_run_id": run_meta_by_run_id,
+ "crawl_summaries": crawl_summaries,
+ }
+
+
+def compute_domain_groups(
+ report_list: list[dict[str, Any]],
+ maps: dict[str, Any],
+ get_payload: Callable[[int], dict[str, Any] | None],
+) -> list[dict[str, Any]]:
+ start_url_by_run_id: dict[int, str] = maps["start_url_by_run_id"]
+ run_created_at_by_run_id: dict[str, str] = maps["run_created_at_by_run_id"]
+ run_meta_by_run_id: dict[int, dict[str, Any]] = maps["run_meta_by_run_id"]
+ brand_map: dict[str, dict[str, Any]] = {}
+
+ for r in report_list:
+ report_id = int(r["id"])
+ payload = get_payload(report_id)
+ if not payload:
+ continue
+
+ run_id = payload.get("crawl_run_id")
+ run_id_int = int(run_id) if run_id is not None else None
+ run_start_url = start_url_by_run_id.get(run_id_int, "") if run_id_int is not None else ""
+ top = payload.get("top_pages") or []
+ links = payload.get("links") or []
+ if top and isinstance(top[0], dict):
+ fallback_url = str(top[0].get("url") or "")
+ elif links and isinstance(links[0], dict):
+ fallback_url = str(links[0].get("url") or "")
+ else:
+ fallback_url = ""
+ crawl_url = (run_start_url or fallback_url or "").strip()
+ start_domain = _extract_hostname(run_start_url)
+ fallback_domain = _extract_hostname(crawl_url)
+ domain_name = start_domain or fallback_domain or str(payload.get("site_name") or UNKNOWN_BRAND)
+ brand_key = start_domain or (f"fallback:{fallback_domain}" if fallback_domain else f"report:{report_id}")
+
+ summary = payload.get("summary") or {}
+ status_counts = {
+ "s2xx": int(summary.get("count_2xx") or 0),
+ "s3xx": int(summary.get("count_3xx") or 0),
+ "s4xx": int(summary.get("count_4xx") or 0),
+ "s5xx": int(summary.get("count_5xx") or 0),
+ "other": int(summary.get("count_error") or 0),
+ }
+ url_count = _crawled_url_count(payload)
+ success_pct = round((status_counts["s2xx"] / url_count) * 100) if url_count > 0 else 0
+ health_score = _score_from_categories(payload.get("categories") or []) or 0
+ run_created_at = run_created_at_by_run_id.get(run_id_int, "") if run_id_int is not None else ""
+ last_crawl = _to_display_datetime(
+ run_created_at or payload.get("crawl_run_created_at") or payload.get("report_generated_at") or r.get("generated_at")
+ )
+ last_audit = _to_display_datetime(payload.get("report_generated_at") or r.get("generated_at"))
+ generated_at_ms = _generated_at_ms(r.get("generated_at"))
+ issue_counts, total_issues = _issue_counts_from_payload(payload)
+ perf_score, seo_score = _lh_scores(payload)
+ technical_seo_score = _category_score(payload, "technical_seo")
+ success_rate_raw = summary.get("success_rate")
+ success_rate = (
+ round(float(success_rate_raw))
+ if isinstance(success_rate_raw, (int, float))
+ else (success_pct if url_count > 0 else None)
+ )
+ crawl_duration_s = (
+ round(float(summary["crawl_time_s"]))
+ if isinstance(summary.get("crawl_time_s"), (int, float))
+ else None
+ )
+ run_meta = run_meta_by_run_id.get(run_id_int) if run_id_int is not None else None
+ canonical_host = _canonical_domain_from_payload(payload, start_url_by_run_id) or _slugify_domain(
+ str(payload.get("site_name") or "")
+ )
+ data_sources = _data_sources(payload)
+
+ group = {
+ "domainName": domain_name,
+ "crawlUrl": crawl_url or EM_DASH,
+ "urlCount": url_count,
+ "healthScore": health_score,
+ "statusCounts": status_counts,
+ "lastCrawl": last_crawl,
+ "lastAudit": last_audit,
+ "totalIssues": total_issues,
+ "issueCounts": issue_counts,
+ "successRate": success_rate,
+ "titleCoverage": None,
+ "avgWordCount": None,
+ "thinPages": None,
+ "technicalSeoScore": technical_seo_score,
+ "perfScore": perf_score,
+ "seoScore": seo_score,
+ "crawlDurationS": crawl_duration_s,
+ "categorySnapshots": _category_snapshots(payload),
+ "seoSignals": _seo_signals(payload),
+ "securityFindings": len(payload.get("security_findings") or []),
+ "duplicateClusters": len(payload.get("content_duplicates") or []),
+ "medianWordCount": _median_word_count(payload),
+ "medianResponseMs": _median_response_ms(payload),
+ "reportId": report_id,
+ "crawlRunId": run_id_int,
+ "generatedAtMs": generated_at_ms,
+ "domainParam": canonical_host,
+ "crawlConfig": _crawl_config_from_payload(payload, run_meta),
+ "dataSources": data_sources,
+ }
+
+ existing = brand_map.get(brand_key)
+ if not existing or generated_at_ms > existing["generatedAtMs"]:
+ brand_map[brand_key] = group
+
+ return sorted(brand_map.values(), key=lambda g: g["generatedAtMs"], reverse=True)
+
+
+def compute_crawl_only_groups(
+ crawl_summaries: list[dict[str, Any]],
+ report_groups: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+ covered_domains = {
+ (g.get("domainParam") or _extract_hostname(g.get("crawlUrl")) or g.get("domainName", "")).lower()
+ for g in report_groups
+ if g.get("domainParam") or g.get("crawlUrl") or g.get("domainName")
+ }
+ covered_run_ids = {
+ int(g["crawlRunId"])
+ for g in report_groups
+ if g.get("crawlRunId") is not None
+ }
+
+ brand_map: dict[str, dict[str, Any]] = {}
+ for row in crawl_summaries:
+ crawl_run_id = int(row["crawl_run_id"])
+ if crawl_run_id in covered_run_ids:
+ continue
+ start_url = str(row.get("start_url") or "").strip()
+ domain_name = _extract_hostname(start_url) or UNKNOWN_BRAND
+ domain_key = domain_name.lower()
+ if not domain_key or domain_key in covered_domains:
+ continue
+
+ url_count = int(row.get("url_count") or 0)
+ with_title = int(row.get("with_title") or 0)
+ title_coverage = _title_coverage_pct(with_title, url_count)
+ avg_word_count = round(float(row.get("avg_word_count") or 0))
+ thin_pages = int(row.get("thin_pages") or 0)
+ generated_at_ms = _generated_at_ms(row.get("created_at"))
+
+ existing = brand_map.get(domain_key)
+ if existing and generated_at_ms <= existing["generatedAtMs"]:
+ continue
+
+ brand_map[domain_key] = {
+ "domainName": domain_name,
+ "crawlUrl": start_url or EM_DASH,
+ "urlCount": url_count,
+ "healthScore": title_coverage,
+ "statusCounts": {
+ "s2xx": int(row.get("s2xx") or 0),
+ "s3xx": int(row.get("s3xx") or 0),
+ "s4xx": int(row.get("s4xx") or 0),
+ "s5xx": int(row.get("s5xx") or 0),
+ "other": int(row.get("other") or 0),
+ },
+ "lastCrawl": _to_display_datetime(row.get("created_at")),
+ "lastAudit": "",
+ "totalIssues": 0,
+ "issueCounts": dict(EMPTY_ISSUE_COUNTS),
+ "successRate": None,
+ "titleCoverage": title_coverage,
+ "avgWordCount": avg_word_count,
+ "thinPages": thin_pages,
+ "technicalSeoScore": None,
+ "perfScore": None,
+ "seoScore": None,
+ "crawlDurationS": None,
+ "categorySnapshots": [],
+ "seoSignals": None,
+ "securityFindings": 0,
+ "duplicateClusters": 0,
+ "medianWordCount": avg_word_count or None,
+ "medianResponseMs": None,
+ "reportId": None,
+ "crawlRunId": crawl_run_id,
+ "crawlOnly": True,
+ "generatedAtMs": generated_at_ms,
+ "domainParam": domain_key,
+ "crawlConfig": _crawl_config_from_summary(row),
+ }
+
+ return list(brand_map.values())
+
+
+def merge_portfolio_groups(
+ report_groups: list[dict[str, Any]],
+ crawl_only_groups: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+ return sorted(
+ report_groups + crawl_only_groups,
+ key=lambda g: g["generatedAtMs"],
+ reverse=True,
+ )
+
+
+def build_crawl_history_by_domain(
+ summaries: list[dict[str, Any]],
+) -> dict[str, list[dict[str, Any]]]:
+ by_domain: dict[str, list[dict[str, Any]]] = {}
+ for row in summaries:
+ key = _extract_hostname(row.get("start_url"))
+ if not key:
+ continue
+ pages = int(row.get("url_count") or 0)
+ point = {
+ "pagesDiscovered": pages,
+ "titleCoverage": _title_coverage_pct(int(row.get("with_title") or 0), pages),
+ "avgWordCount": round(float(row.get("avg_word_count") or 0)),
+ "createdAtMs": _generated_at_ms(row.get("created_at")),
+ }
+ by_domain.setdefault(key, []).append(point)
+
+ out: dict[str, list[dict[str, Any]]] = {}
+ for key, points in by_domain.items():
+ out[key] = sorted(points, key=lambda p: p["createdAtMs"])[-8:]
+ return out
+
+
+def compute_portfolio_summary(groups: list[dict[str, Any]]) -> dict[str, Any]:
+ total_brands = len(groups)
+ total_urls = sum(int(g.get("urlCount") or 0) for g in groups)
+ avg_health = (
+ round(sum(int(g.get("healthScore") or 0) for g in groups) / total_brands)
+ if total_brands
+ else None
+ )
+ return {"totalBrands": total_brands, "totalUrls": total_urls, "avgHealth": avg_health}
+
+
+def build_portfolio_card(
+ conn: Connection,
+ report_list: list[dict[str, Any]],
+ maps: dict[str, Any],
+ *,
+ report_id: int | None = None,
+ crawl_run_id: int | None = None,
+) -> dict[str, Any] | None:
+ def get_full_payload(rid: int) -> dict[str, Any] | None:
+ return read_report_payload(conn, rid)
+
+ if report_id is not None:
+ row = next((r for r in report_list if int(r["id"]) == report_id), None)
+ if not row:
+ return None
+ groups = compute_domain_groups([row], maps, get_full_payload)
+ return groups[0] if groups else None
+
+ if crawl_run_id is not None:
+ report_groups = compute_domain_groups(report_list, maps, get_full_payload)
+ from_report = next((g for g in report_groups if g.get("crawlRunId") == crawl_run_id), None)
+ if from_report:
+ return from_report
+ summary = next(
+ (s for s in maps["crawl_summaries"] if int(s["crawl_run_id"]) == crawl_run_id),
+ None,
+ )
+ if not summary:
+ return None
+ crawl_only = compute_crawl_only_groups([summary], report_groups)
+ return crawl_only[0] if crawl_only else None
+
+ return None
+
+
+def build_groups_bundle(
+ conn: Connection,
+ report_list: list[dict[str, Any]],
+ *,
+ lite: bool,
+) -> dict[str, Any]:
+ maps = load_portfolio_maps(conn)
+
+ def get_payload(rid: int) -> dict[str, Any] | None:
+ payload = read_report_payload(conn, rid)
+ if payload is None:
+ return None
+ return slice_payload_for_section(payload, "core") if lite else payload
+
+ report_groups = compute_domain_groups(report_list, maps, get_payload)
+ crawl_only = compute_crawl_only_groups(maps["crawl_summaries"], report_groups)
+ groups = merge_portfolio_groups(report_groups, crawl_only)
+ crawl_history = build_crawl_history_by_domain(maps["crawl_summaries"])
+ return {"groups": groups, "crawlHistoryByDomain": crawl_history}
+
+
+def get_portfolio_response(
+ conn: Connection,
+ *,
+ widget: str,
+ ids: list[int],
+ report_id: int | None = None,
+ crawl_run_id: int | None = None,
+) -> dict[str, Any]:
+ all_reports = list_reports(conn)
+ id_set = set(ids)
+ report_list = [r for r in all_reports if r["id"] in id_set] if ids else all_reports
+
+ if widget == "card":
+ maps = load_portfolio_maps(conn)
+ group = build_portfolio_card(
+ conn,
+ report_list,
+ maps,
+ report_id=report_id,
+ crawl_run_id=crawl_run_id,
+ )
+ return {"group": group}
+
+ lite = widget in ("groups", "summary")
+ bundle = build_groups_bundle(conn, report_list, lite=lite)
+
+ if widget == "summary":
+ return compute_portfolio_summary(bundle["groups"])
+
+ return {
+ "groups": bundle["groups"],
+ "crawlHistoryByDomain": bundle["crawlHistoryByDomain"],
+ }
diff --git a/src/website_profiling/api/services/report_loader.py b/src/website_profiling/api/services/report_loader.py
new file mode 100644
index 00000000..18dcd440
--- /dev/null
+++ b/src/website_profiling/api/services/report_loader.py
@@ -0,0 +1,382 @@
+"""Report data loading service — DB queries for the /api/report/* routes."""
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from psycopg import Connection
+
+from website_profiling.db._common import _parse_row_json, _row_field
+from website_profiling.db.report_store import read_report_payload
+
+# ── Section slicing ─────────────────────────────────────────────────────────
+
+SECTION_FIELDS: dict[str, list[str]] = {
+ "core": [
+ "site_name", "summary", "categories", "top_pages", "recommendations",
+ "seo_health", "social_coverage", "status_counts", "portfolio_benchmark",
+ "executive_summary", "crux_summary", "report_meta", "report_generated_at",
+ "crawl_only_preview", "crawl_run_id", "crawl_run_created_at", "site_level",
+ "ml_errors",
+ ],
+ "links": [
+ "links", "link_edges", "link_rel_summary", "inlink_anchor_matrix",
+ "outbound_link_domains", "outlink_labels", "outlink_counts",
+ ],
+ "traffic": ["google"],
+ "keywords": [
+ "keywords", "keyword_opportunities", "competitor_keyword_gap",
+ "semantic_keyword_clusters",
+ ],
+ "issues": ["issues", "redirects"],
+ "content": [
+ "content_urls", "content_duplicates", "content_analytics",
+ "text_content_analysis", "response_time_stats",
+ ],
+ "lighthouse": [
+ "lighthouse_summary", "lighthouse_by_url", "lighthouse_diagnostics",
+ "lighthouse_human_summary",
+ ],
+ "security": ["security_findings"],
+ "gsc-links": ["gsc_links", "bing_backlinks"],
+ "structure": ["graph_nodes", "graph_edges", "depth_distribution"],
+ "tech": ["tech_stack_summary", "subdomains", "contact_intelligence"],
+ "indexation": [
+ "indexation_coverage", "hreflang_summary", "ner_site_summary",
+ "language_summary", "rich_results_validation", "url_fingerprints",
+ "rich_results_meta",
+ ],
+ "gallery": [
+ "mime_labels", "mime_values", "title_labels", "title_counts",
+ "domain_labels", "domain_values",
+ ],
+}
+
+SECTION_KEYS = list(SECTION_FIELDS.keys())
+
+
+def slice_payload_for_section(
+ payload: dict[str, Any], section: str
+) -> dict[str, Any]:
+ fields = SECTION_FIELDS.get(section, [])
+ return {k: payload[k] for k in fields if k in payload}
+
+
+# ── Report list ──────────────────────────────────────────────────────────────
+
+def list_reports(conn: Connection) -> list[dict[str, Any]]:
+ cur = conn.execute(
+ "SELECT id, canonical_domain, site_name, generated_at FROM report_payload ORDER BY id DESC"
+ )
+ rows = cur.fetchall()
+ result = []
+ for row in rows:
+ generated = _row_field(row, "generated_at")
+ result.append({
+ "id": int(_row_field(row, "id")),
+ "canonical_domain": _row_field(row, "canonical_domain"),
+ "site_name": _row_field(row, "site_name"),
+ "generated_at": generated.isoformat() if hasattr(generated, "isoformat") else generated,
+ })
+ return result
+
+
+# ── Crawl runs ───────────────────────────────────────────────────────────────
+
+def list_crawl_runs(conn: Connection) -> list[dict[str, Any]]:
+ try:
+ cur = conn.execute(
+ "SELECT id, start_url, created_at, render_mode, discovery_mode FROM crawl_runs ORDER BY id DESC"
+ )
+ rows = cur.fetchall()
+ except Exception:
+ return []
+ result = []
+ for row in rows:
+ created = _row_field(row, "created_at")
+ result.append({
+ "id": int(_row_field(row, "id")),
+ "start_url": str(_row_field(row, "start_url") or ""),
+ "created_at": created.isoformat() if hasattr(created, "isoformat") else str(created or ""),
+ "render_mode": _row_field(row, "render_mode"),
+ "discovery_mode": _row_field(row, "discovery_mode"),
+ })
+ return result
+
+
+def list_crawl_run_summaries(conn: Connection) -> list[dict[str, Any]]:
+ """Aggregate crawl run stats for portfolio cards and crawl history."""
+ try:
+ cur = conn.execute(
+ """
+ SELECT
+ cr.id AS crawl_run_id,
+ cr.start_url,
+ cr.created_at,
+ cr.render_mode,
+ cr.discovery_mode,
+ COUNT(crl.id)::int AS url_count,
+ COUNT(*) FILTER (WHERE crl.status LIKE '2%%')::int AS s2xx,
+ COUNT(*) FILTER (WHERE crl.status LIKE '3%%')::int AS s3xx,
+ COUNT(*) FILTER (WHERE crl.status LIKE '4%%')::int AS s4xx,
+ COUNT(*) FILTER (WHERE crl.status LIKE '5%%')::int AS s5xx,
+ COUNT(*) FILTER (
+ WHERE crl.status IS NULL
+ OR crl.status = ''
+ OR crl.status !~ '^[2345]'
+ )::int AS other,
+ COUNT(*) FILTER (
+ WHERE NULLIF(TRIM(COALESCE(crl.title, crl.data->>'title', '')), '') IS NOT NULL
+ )::int AS with_title,
+ COALESCE(ROUND(AVG(NULLIF((crl.data->>'word_count')::numeric, 0))), 0)::int AS avg_word_count,
+ COUNT(*) FILTER (
+ WHERE COALESCE((crl.data->>'word_count')::int, 0) > 0
+ AND COALESCE((crl.data->>'word_count')::int, 0) < 300
+ )::int AS thin_pages
+ FROM crawl_runs cr
+ LEFT JOIN crawl_results crl ON crl.crawl_run_id = cr.id
+ GROUP BY cr.id, cr.start_url, cr.created_at, cr.render_mode, cr.discovery_mode
+ ORDER BY cr.id DESC
+ """
+ )
+ rows = cur.fetchall()
+ except Exception:
+ return []
+ result = []
+ for row in rows:
+ created = _row_field(row, "created_at")
+ result.append({
+ "crawl_run_id": int(_row_field(row, "crawl_run_id")),
+ "start_url": str(_row_field(row, "start_url") or ""),
+ "created_at": created.isoformat() if hasattr(created, "isoformat") else str(created or ""),
+ "url_count": int(_row_field(row, "url_count") or 0),
+ "s2xx": int(_row_field(row, "s2xx") or 0),
+ "s3xx": int(_row_field(row, "s3xx") or 0),
+ "s4xx": int(_row_field(row, "s4xx") or 0),
+ "s5xx": int(_row_field(row, "s5xx") or 0),
+ "other": int(_row_field(row, "other") or 0),
+ "with_title": int(_row_field(row, "with_title") or 0),
+ "avg_word_count": int(_row_field(row, "avg_word_count") or 0),
+ "thin_pages": int(_row_field(row, "thin_pages") or 0),
+ "render_mode": _row_field(row, "render_mode"),
+ "discovery_mode": _row_field(row, "discovery_mode"),
+ })
+ return result
+
+
+# ── Report payload ───────────────────────────────────────────────────────────
+
+def get_report_payload(
+ conn: Connection,
+ report_id: Optional[int] = None,
+ domain: Optional[str] = None,
+ section: Optional[str] = None,
+) -> Optional[dict[str, Any]]:
+ resolved_id = report_id
+
+ if resolved_id is None and domain:
+ domain_lower = domain.strip().lower()
+ reports = list_reports(conn)
+ match = next(
+ (r for r in reports if (r.get("canonical_domain") or "").lower() == domain_lower),
+ None,
+ )
+ if match:
+ resolved_id = match["id"]
+
+ payload = read_report_payload(conn, resolved_id)
+ if payload is None:
+ return None
+
+ if section and section in SECTION_FIELDS:
+ return slice_payload_for_section(payload, section)
+ return payload
+
+
+# ── Crawl preview ────────────────────────────────────────────────────────────
+
+def get_crawl_preview_payload(conn: Connection, crawl_run_id: int) -> dict[str, Any]:
+ cur = conn.execute(
+ "SELECT id, start_url, created_at FROM crawl_runs WHERE id = %s",
+ (crawl_run_id,),
+ )
+ run_row = cur.fetchone()
+ if not run_row:
+ raise ValueError("Crawl run not found")
+
+ start_url = str(_row_field(run_row, "start_url") or "")
+ from urllib.parse import urlparse
+ try:
+ site_host = urlparse(start_url).hostname or ""
+ except Exception:
+ site_host = ""
+
+ cur2 = conn.execute(
+ "SELECT url, data FROM crawl_results WHERE crawl_run_id = %s",
+ (crawl_run_id,),
+ )
+ pages = []
+ for row in cur2.fetchall():
+ data = _parse_row_json(row, "data", index=1)
+ if not isinstance(data, dict):
+ data = {}
+ pages.append({"url": str(_row_field(row, "url") or ""), **data})
+
+ return {
+ "crawl_only_preview": True,
+ "crawl_run_id": crawl_run_id,
+ "site_name": site_host,
+ "top_pages": pages,
+ }
+
+
+# ── Audit history ────────────────────────────────────────────────────────────
+
+def _avg_score(categories: list[dict[str, Any]]) -> Optional[int]:
+ nums = [float(c["score"]) for c in categories if isinstance(c.get("score"), (int, float))]
+ if not nums:
+ return None
+ return round(sum(nums) / len(nums))
+
+
+def _issue_counts(categories: list[dict[str, Any]]) -> dict[str, int]:
+ counts: dict[str, int] = {"Critical": 0, "High": 0, "Medium": 0, "Low": 0}
+ for cat in categories:
+ for issue in (cat.get("issues") or []):
+ p = str(issue.get("priority") or "Medium")
+ counts[p] = counts.get(p, 0) + 1
+ return counts
+
+
+def _lh_scores(payload: dict[str, Any]) -> tuple[Optional[int], Optional[int]]:
+ summary = payload.get("lighthouse_summary")
+ if not isinstance(summary, dict):
+ return None, None
+ mm = summary.get("median_metrics") or {}
+ cs = summary.get("category_scores") or {}
+ perf_raw = mm.get("performance_score") or cs.get("performance")
+ seo_raw = mm.get("seo_score") or cs.get("seo")
+ perf = round(float(perf_raw)) if isinstance(perf_raw, (int, float)) else None
+ seo = round(float(seo_raw)) if isinstance(seo_raw, (int, float)) else None
+ return perf, seo
+
+
+def list_audit_history(
+ conn: Connection,
+ property_id: Optional[int] = None,
+ domain: Optional[str] = None,
+ limit: int = 20,
+) -> list[dict[str, Any]]:
+ clauses: list[str] = []
+ vals: list[Any] = []
+
+ if property_id is not None and property_id > 0:
+ clauses.append("property_id = %s")
+ vals.append(property_id)
+ elif domain:
+ normalized = domain.strip().lower()
+ clauses.append(
+ "(LOWER(canonical_domain) = %s OR regexp_replace(LOWER(COALESCE(canonical_domain, '')), '[^a-z0-9]+', '-', 'g') = %s)"
+ )
+ vals.append(normalized)
+ vals.append(normalized)
+
+ limit = max(1, min(100, limit))
+ vals.append(limit)
+ where = f"WHERE {' AND '.join(clauses)}" if clauses else ""
+
+ cur = conn.execute(
+ f"""SELECT id, canonical_domain, site_name, generated_at, data
+ FROM report_payload {where}
+ ORDER BY generated_at DESC LIMIT %s""",
+ vals,
+ )
+ rows = cur.fetchall()
+ result = []
+ for row in rows:
+ data = _parse_row_json(row, "data")
+ if not isinstance(data, dict):
+ data = {}
+ categories = data.get("categories") or []
+ cat_scores = {
+ (c.get("id") or c.get("name") or "unknown"): float(c["score"])
+ for c in categories
+ if isinstance(c.get("score"), (int, float))
+ }
+ perf, seo = _lh_scores(data)
+ tech_seo_cat = next((c for c in categories if c.get("id") == "technical_seo"), None)
+ tech_seo = round(float(tech_seo_cat["score"])) if tech_seo_cat and isinstance(tech_seo_cat.get("score"), (int, float)) else None
+ generated_at = _row_field(row, "generated_at")
+ result.append({
+ "reportId": int(_row_field(row, "id")),
+ "canonicalDomain": _row_field(row, "canonical_domain"),
+ "siteName": _row_field(row, "site_name"),
+ "generatedAt": generated_at.isoformat() if hasattr(generated_at, "isoformat") else generated_at,
+ "healthScore": _avg_score(categories),
+ "categoryScores": cat_scores,
+ "issueCounts": _issue_counts(categories),
+ "perfScore": perf,
+ "seoScore": seo,
+ "technicalSeoScore": tech_seo,
+ })
+ return result
+
+
+# ── Mobile-desktop delta ─────────────────────────────────────────────────────
+
+def get_mobile_desktop_delta(conn: Connection, run_id: int) -> list[dict[str, Any]]:
+ cur = conn.execute(
+ "SELECT mobile_run_id FROM crawl_runs WHERE id = %s", (run_id,)
+ )
+ row = cur.fetchone()
+ mobile_run_id = _row_field(row, "mobile_run_id")
+ if not row or mobile_run_id is None:
+ return []
+ mobile_run_id = int(mobile_run_id)
+
+ def fetch_run(rid: int) -> dict[str, dict[str, Any]]:
+ c = conn.execute(
+ "SELECT url, data FROM crawl_results WHERE crawl_run_id = %s", (rid,)
+ )
+ m: dict[str, dict[str, Any]] = {}
+ for r in c.fetchall():
+ d = _parse_row_json(r, "data", index=1)
+ if not isinstance(d, dict):
+ d = {}
+ key = str(_row_field(r, "url") or "").rstrip("/").lower()
+ m[key] = {
+ "title": str(d.get("title") or ""),
+ "h1": str(d.get("h1") or ""),
+ "word_count": int(d.get("word_count") or 0),
+ "status": int(d.get("status") or 0),
+ }
+ return m
+
+ desktop_map = fetch_run(run_id)
+ mobile_map = fetch_run(mobile_run_id)
+
+ deltas = []
+ for key, desktop in desktop_map.items():
+ mobile = mobile_map.get(key)
+ if not mobile:
+ continue
+ title_differs = desktop["title"] != mobile["title"]
+ h1_differs = desktop["h1"] != mobile["h1"]
+ word_count_delta = abs(desktop["word_count"] - mobile["word_count"])
+ status_differs = desktop["status"] != mobile["status"]
+ if not title_differs and not h1_differs and word_count_delta <= 50 and not status_differs:
+ continue
+ deltas.append({
+ "url": key,
+ "desktop": desktop,
+ "mobile": mobile,
+ "title_differs": title_differs,
+ "h1_differs": h1_differs,
+ "word_count_delta": word_count_delta,
+ "status_differs": status_differs,
+ })
+
+ deltas.sort(
+ key=lambda d: (d["status_differs"] * 4 + d["title_differs"] * 2 + d["h1_differs"]),
+ reverse=True,
+ )
+ return deltas
diff --git a/src/website_profiling/clients/__init__.py b/src/website_profiling/clients/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/src/website_profiling/clients/file_service.py b/src/website_profiling/clients/file_service.py
new file mode 100644
index 00000000..3285c3b1
--- /dev/null
+++ b/src/website_profiling/clients/file_service.py
@@ -0,0 +1,59 @@
+"""HTTP client for FileService (.NET) — PDF and workbook exports."""
+from __future__ import annotations
+
+import os
+from typing import Optional
+from urllib.parse import urlencode
+
+import requests
+
+_DEFAULT_BASE = "http://127.0.0.1:8080"
+_TIMEOUT_SECONDS = 120
+
+
+def _base_url() -> str:
+ return (os.environ.get("FILE_SERVICE_URL") or _DEFAULT_BASE).strip().rstrip("/")
+
+
+def _get_bytes(path: str, *, params: Optional[dict[str, str]] = None) -> bytes:
+ url = f"{_base_url()}{path}"
+ if params:
+ url = f"{url}?{urlencode(params)}"
+ try:
+ response = requests.get(url, timeout=_TIMEOUT_SECONDS)
+ except requests.RequestException as exc:
+ raise RuntimeError(f"File service unreachable at {_base_url()}: {exc}") from exc
+ if response.status_code == 404:
+ raise FileNotFoundError(response.text or "Report not found")
+ if response.status_code >= 400:
+ raise RuntimeError(
+ f"File service returned {response.status_code}: {response.text[:500]}"
+ )
+ return response.content
+
+
+def fetch_report_pdf(
+ report_id: Optional[int] = None,
+ *,
+ profile: str = "standard",
+ branding: bool = True,
+) -> bytes:
+ """Fetch audit PDF bytes from FileService."""
+ params = {
+ "profile": profile,
+ "disposition": "attachment",
+ "branding": "true" if branding else "false",
+ }
+ if report_id is not None:
+ return _get_bytes(f"/v1/reports/{int(report_id)}/pdf", params=params)
+ raise ValueError("report_id is required for PDF export")
+
+
+def fetch_report_workbook(report_id: Optional[int] = None) -> bytes:
+ """Fetch crawl workbook (.xlsx) bytes from FileService."""
+ if report_id is None:
+ raise ValueError("report_id is required for workbook export")
+ return _get_bytes(
+ f"/v1/reports/{int(report_id)}/workbook",
+ params={"disposition": "attachment"},
+ )
diff --git a/src/website_profiling/commands/pipeline_cmd.py b/src/website_profiling/commands/pipeline_cmd.py
index 967468bd..06853905 100644
--- a/src/website_profiling/commands/pipeline_cmd.py
+++ b/src/website_profiling/commands/pipeline_cmd.py
@@ -97,13 +97,13 @@ def select_lighthouse_urls_from_gsc(
if not google_data or max_pages <= 0:
return []
gsc = google_data.get("gsc") if isinstance(google_data.get("gsc"), dict) else {}
- pages = gsc.get("pages") if isinstance(gsc.get("pages"), list) else []
+ pages = gsc.get("top_pages") if isinstance(gsc.get("top_pages"), list) else []
crawl_set = {u.rstrip("/") for u in crawl_urls}
ranked: list[tuple[float, str]] = []
for row in pages:
if not isinstance(row, dict):
continue
- url = str(row.get("page") or row.get("url") or "").strip()
+ url = str(row.get("page") or "").strip()
if not url:
continue
norm = url.rstrip("/")
diff --git a/src/website_profiling/db/config_store.py b/src/website_profiling/db/config_store.py
index 7bcc10cd..856ca49b 100644
--- a/src/website_profiling/db/config_store.py
+++ b/src/website_profiling/db/config_store.py
@@ -1,11 +1,6 @@
"""Pipeline and LLM config tables."""
from __future__ import annotations
-import json
-import os
-import subprocess
-import time
-from pathlib import Path
from typing import Any, Optional
import pandas as pd
@@ -17,6 +12,7 @@
_json_val,
_now_iso,
_parse_json_field,
+ _row_field,
_sanitize_for_json,
)
from .pool import db_session, get_data_dir, get_database_url
@@ -82,3 +78,41 @@ def write_llm_config(conn: Connection, entries: dict[str, str], secret_keys: set
)
+def read_llm_config_full(conn: Connection) -> list[dict[str, Any]]:
+ """Return llm_config rows including the is_secret flag."""
+ try:
+ cur = conn.execute("SELECT key, value, is_secret FROM llm_config ORDER BY key")
+ return [
+ {
+ "key": str(_row_field(row, "key", index=0)),
+ "value": str(_row_field(row, "value", index=1)),
+ "is_secret": bool(_row_field(row, "is_secret", index=2)),
+ }
+ for row in cur.fetchall() or []
+ ]
+ except Exception:
+ return []
+
+
+def read_app_setting(conn: Connection, key: str) -> str | None:
+ try:
+ cur = conn.execute("SELECT value FROM app_settings WHERE key = %s", (key,))
+ row = cur.fetchone()
+ if not row:
+ return None
+ val = _row_field(row, "value", index=0)
+ return str(val) if val is not None else None
+ except Exception:
+ return None
+
+
+def write_app_setting(conn: Connection, key: str, value: str) -> None:
+ conn.execute(
+ """INSERT INTO app_settings (key, value, updated_at)
+ VALUES (%s, %s, now())
+ ON CONFLICT (key) DO UPDATE
+ SET value = EXCLUDED.value,
+ updated_at = now()""",
+ (key, value),
+ )
+ conn.commit()
diff --git a/src/website_profiling/db/content_draft_store.py b/src/website_profiling/db/content_draft_store.py
new file mode 100644
index 00000000..41f41118
--- /dev/null
+++ b/src/website_profiling/db/content_draft_store.py
@@ -0,0 +1,177 @@
+"""Content drafts for Content Studio (content_drafts table)."""
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from psycopg import Connection
+from psycopg.types.json import Json
+
+from ._common import _parse_row_json, _row_field
+
+_LIST_COLUMNS = """
+ id, property_id, title, target_keyword, landing_url, status,
+ grade_score, created_at::text, updated_at::text
+"""
+
+_DETAIL_COLUMNS = """
+ id, property_id, title, target_keyword, landing_url, status,
+ body_html, title_tag, meta_description, grade_score, grade_snapshot,
+ created_at::text, updated_at::text
+"""
+
+
+def _grade_score_value(raw: Any) -> float | None:
+ if raw is None:
+ return None
+ return float(raw)
+
+
+def _map_list_row(row: Any) -> dict[str, Any]:
+ return {
+ "id": int(_row_field(row, "id")),
+ "property_id": int(_row_field(row, "property_id")),
+ "title": _row_field(row, "title"),
+ "target_keyword": _row_field(row, "target_keyword"),
+ "landing_url": _row_field(row, "landing_url"),
+ "status": _row_field(row, "status"),
+ "grade_score": _grade_score_value(_row_field(row, "grade_score")),
+ "created_at": _row_field(row, "created_at"),
+ "updated_at": _row_field(row, "updated_at"),
+ }
+
+
+def _map_detail_row(row: Any) -> dict[str, Any]:
+ return {
+ "id": int(_row_field(row, "id")),
+ "property_id": int(_row_field(row, "property_id")),
+ "title": _row_field(row, "title"),
+ "target_keyword": _row_field(row, "target_keyword"),
+ "landing_url": _row_field(row, "landing_url"),
+ "status": _row_field(row, "status"),
+ "body_html": _row_field(row, "body_html") or "",
+ "title_tag": _row_field(row, "title_tag") or "",
+ "meta_description": _row_field(row, "meta_description") or "",
+ "grade_score": _grade_score_value(_row_field(row, "grade_score")),
+ "grade_snapshot": _parse_row_json(row, "grade_snapshot"),
+ "created_at": _row_field(row, "created_at"),
+ "updated_at": _row_field(row, "updated_at"),
+ }
+
+
+def list_content_drafts(
+ conn: Connection,
+ property_id: int,
+ *,
+ limit: int = 100,
+) -> list[dict[str, Any]]:
+ limit = max(1, min(int(limit), 200))
+ cur = conn.execute(
+ f"""SELECT {_LIST_COLUMNS}
+ FROM content_drafts
+ WHERE property_id = %s
+ ORDER BY updated_at DESC
+ LIMIT %s""",
+ (property_id, limit),
+ )
+ return [_map_list_row(row) for row in cur.fetchall() or []]
+
+
+def get_content_draft(conn: Connection, draft_id: int) -> dict[str, Any] | None:
+ cur = conn.execute(
+ f"SELECT {_DETAIL_COLUMNS} FROM content_drafts WHERE id = %s",
+ (draft_id,),
+ )
+ row = cur.fetchone()
+ return _map_detail_row(row) if row else None
+
+
+def create_content_draft(
+ conn: Connection,
+ property_id: int,
+ *,
+ title: str = "Untitled draft",
+ target_keyword: str = "",
+ landing_url: str | None = None,
+ status: str = "draft",
+ body_html: str = "",
+ title_tag: str = "",
+ meta_description: str = "",
+) -> int:
+ cur = conn.execute(
+ """INSERT INTO content_drafts
+ (property_id, title, target_keyword, landing_url, status,
+ body_html, title_tag, meta_description)
+ VALUES (%s, %s, %s, %s, %s, %s, %s, %s)
+ RETURNING id""",
+ (
+ property_id,
+ (title or "Untitled draft").strip() or "Untitled draft",
+ (target_keyword or "").strip(),
+ (landing_url or "").strip() or None,
+ status or "draft",
+ body_html or "",
+ title_tag or "",
+ meta_description or "",
+ ),
+ )
+ row = cur.fetchone()
+ conn.commit()
+ return int(_row_field(row, "id"))
+
+
+def update_content_draft(
+ conn: Connection,
+ draft_id: int,
+ patch: dict[str, Any],
+) -> dict[str, Any] | None:
+ fields: list[str] = []
+ values: list[Any] = []
+
+ def set_field(col: str, val: Any) -> None:
+ fields.append(f"{col} = %s")
+ values.append(val)
+
+ if "title" in patch:
+ set_field("title", (str(patch["title"]).strip() or "Untitled draft"))
+ if "target_keyword" in patch:
+ set_field("target_keyword", str(patch["target_keyword"]).strip())
+ if "landing_url" in patch:
+ set_field("landing_url", str(patch["landing_url"]).strip() or None)
+ if "status" in patch:
+ set_field("status", patch["status"])
+ if "body_html" in patch:
+ set_field("body_html", patch["body_html"])
+ if "title_tag" in patch:
+ set_field("title_tag", patch["title_tag"])
+ if "meta_description" in patch:
+ set_field("meta_description", patch["meta_description"])
+ if "grade_score" in patch:
+ set_field("grade_score", patch["grade_score"])
+ if "grade_snapshot" in patch:
+ gs = patch["grade_snapshot"]
+ set_field("grade_snapshot", Json(gs) if gs is not None else None)
+
+ if not fields:
+ return get_content_draft(conn, draft_id)
+
+ fields.append("updated_at = now()")
+ values.append(draft_id)
+ cur = conn.execute(
+ f"""UPDATE content_drafts SET {', '.join(fields)}
+ WHERE id = %s
+ RETURNING {_DETAIL_COLUMNS}""",
+ values,
+ )
+ row = cur.fetchone()
+ conn.commit()
+ return _map_detail_row(row) if row else None
+
+
+def delete_content_draft(conn: Connection, draft_id: int) -> bool:
+ cur = conn.execute(
+ "DELETE FROM content_drafts WHERE id = %s RETURNING id",
+ (draft_id,),
+ )
+ deleted = cur.fetchone() is not None
+ conn.commit()
+ return deleted
diff --git a/src/website_profiling/db/crawl_store.py b/src/website_profiling/db/crawl_store.py
index 5546368e..263a7d17 100644
--- a/src/website_profiling/db/crawl_store.py
+++ b/src/website_profiling/db/crawl_store.py
@@ -34,36 +34,13 @@ def create_crawl_run(
) -> int:
mode = (render_mode or "static").strip().lower()
disc = (discovery_mode or "spider").strip().lower()
- statements = [
- (
- "INSERT INTO crawl_runs (created_at, start_url, property_id, render_mode, discovery_mode) VALUES (%s, %s, %s, %s, %s) RETURNING id",
- (_now_iso(), start_url, property_id, mode, disc),
- ),
- (
- "INSERT INTO crawl_runs (created_at, start_url, property_id, render_mode) VALUES (%s, %s, %s, %s) RETURNING id",
- (_now_iso(), start_url, property_id, mode),
- ),
- (
- "INSERT INTO crawl_runs (created_at, start_url, property_id) VALUES (%s, %s, %s) RETURNING id",
- (_now_iso(), start_url, property_id),
- ),
- ]
- last_err: Exception | None = None
- for sql, params in statements:
- try:
- cur = conn.execute(sql, params)
- row = cur.fetchone()
- conn.commit()
- return int(row["id"])
- except Exception as exc:
- last_err = exc
- try:
- conn.rollback()
- except Exception:
- pass
- if last_err is not None:
- raise last_err
- raise RuntimeError("create_crawl_run failed") # pragma: no cover
+ cur = conn.execute(
+ "INSERT INTO crawl_runs (created_at, start_url, property_id, render_mode, discovery_mode) VALUES (%s, %s, %s, %s, %s) RETURNING id",
+ (_now_iso(), start_url, property_id, mode, disc),
+ )
+ row = cur.fetchone()
+ conn.commit()
+ return int(row["id"])
def get_latest_crawl_run_id(conn: Connection) -> Optional[int]:
@@ -84,25 +61,13 @@ def get_crawl_run_info(conn: Connection, run_id: int) -> Optional[dict[str, Any]
row = cur.fetchone()
if row is None:
return None
- out: dict[str, Any] = {
+ return {
"created_at": row["created_at"],
"start_url": row["start_url"],
+ "render_mode": row["render_mode"],
}
- if "render_mode" in row.keys():
- out["render_mode"] = row["render_mode"]
- return out
except Exception:
- try:
- cur = conn.execute(
- "SELECT created_at, start_url FROM crawl_runs WHERE id = %s",
- (run_id,),
- )
- row = cur.fetchone()
- if row is None:
- return None
- return {"created_at": row["created_at"], "start_url": row["start_url"]}
- except Exception:
- return None
+ return None
def set_mobile_run_id(conn: Connection, desktop_run_id: int, mobile_run_id: int) -> None:
@@ -289,14 +254,6 @@ def _canonical_domain_from_report(conn: Connection, report_data: dict[str, Any])
fetch_method = EXCLUDED.fetch_method,
data = EXCLUDED.data"""
-_CRAWL_INSERT_SQL_LEGACY = """INSERT INTO crawl_results (crawl_run_id, url, status, title, data)
-VALUES (%s, %s, %s, %s, %s)
-ON CONFLICT (crawl_run_id, url) DO UPDATE SET
- status = EXCLUDED.status,
- title = EXCLUDED.title,
- data = EXCLUDED.data"""
-
-
def _crawl_rows_from_df(df: pd.DataFrame, crawl_run_id: int) -> list[tuple]:
rows: list[tuple] = []
if df.empty or "url" not in df.columns:
@@ -324,24 +281,8 @@ def _crawl_rows_from_df(df: pd.DataFrame, crawl_run_id: int) -> list[tuple]:
def _write_crawl_rows(conn: Connection, rows: list[tuple]) -> None:
if not rows:
return
- normalized: list[tuple] = []
- for row in rows:
- if len(row) == 5:
- normalized.append((row[0], row[1], row[2], row[3], "static", row[4]))
- else:
- normalized.append(row)
- try:
- # Savepoint so that a failure (e.g. a legacy schema missing the
- # fetch_method column) rolls back ONLY this insert and leaves the
- # transaction usable. Without it the legacy fallback below runs inside an
- # aborted transaction, raises "current transaction is aborted", and
- # silently writes nothing.
- with conn.transaction():
- _executemany(conn, _CRAWL_INSERT_SQL, normalized, page_size=_CRAWL_BATCH_SIZE)
- except Exception:
- legacy = [(r[0], r[1], r[2], r[3], r[5]) for r in normalized]
- with conn.transaction():
- _executemany(conn, _CRAWL_INSERT_SQL_LEGACY, legacy, page_size=_CRAWL_BATCH_SIZE)
+ with conn.transaction():
+ _executemany(conn, _CRAWL_INSERT_SQL, rows, page_size=_CRAWL_BATCH_SIZE)
def write_crawl_batch(
@@ -431,36 +372,13 @@ def merge_crawl_result_fields_batch(
def read_crawl(conn: Connection, run_id: Optional[int] = None) -> pd.DataFrame:
- try:
- return _read_crawl_rows(conn, run_id, include_fetch_method=True)
- except Exception:
- try:
- return _read_crawl_rows(conn, run_id, include_fetch_method=False)
- except Exception:
- return pd.DataFrame()
-
-
-def _read_crawl_rows(
- conn: Connection,
- run_id: Optional[int],
- *,
- include_fetch_method: bool,
-) -> pd.DataFrame:
if run_id is None:
run_id = get_latest_crawl_run_id(conn)
- if include_fetch_method:
- if run_id is None:
- cur = conn.execute("SELECT url, fetch_method, data FROM crawl_results")
- else:
- cur = conn.execute(
- "SELECT url, fetch_method, data FROM crawl_results WHERE crawl_run_id = %s",
- (run_id,),
- )
- elif run_id is None:
- cur = conn.execute("SELECT url, data FROM crawl_results")
+ if run_id is None:
+ cur = conn.execute("SELECT url, fetch_method, data FROM crawl_results")
else:
cur = conn.execute(
- "SELECT url, data FROM crawl_results WHERE crawl_run_id = %s",
+ "SELECT url, fetch_method, data FROM crawl_results WHERE crawl_run_id = %s",
(run_id,),
)
rows = cur.fetchall()
@@ -469,19 +387,12 @@ def _read_crawl_rows(
records = []
for row in rows:
rec: dict[str, Any] = {"url": row["url"]}
- fm_col: Optional[str] = None
- if include_fetch_method and "fetch_method" in row.keys():
- fm_col = str(row["fetch_method"] or "static").strip() or "static"
+ if "fetch_method" in row.keys():
+ rec["fetch_method"] = str(row["fetch_method"] or "static").strip() or "static"
data = _parse_row_json(row) or {}
if isinstance(data, dict):
rec.update(data)
- if fm_col is not None:
- rec["fetch_method"] = fm_col
- elif not include_fetch_method:
- rec["fetch_method"] = str(
- (data.get("fetch_method") if isinstance(data, dict) else None) or "static"
- ).strip() or "static"
- elif "fetch_method" not in rec:
+ if "fetch_method" not in rec:
rec["fetch_method"] = "static"
records.append(rec)
df = pd.DataFrame(records)
diff --git a/src/website_profiling/db/dashboard_store.py b/src/website_profiling/db/dashboard_store.py
new file mode 100644
index 00000000..308d03c2
--- /dev/null
+++ b/src/website_profiling/db/dashboard_store.py
@@ -0,0 +1,116 @@
+"""Custom dashboards (dashboards table)."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+from psycopg.types.json import Json
+
+from ._common import _row_field
+
+_SELECT = """
+ SELECT id, property_id, name, layout_json, is_default, created_at, updated_at
+ FROM dashboards
+"""
+
+
+def _map_dashboard(row: Any) -> dict[str, Any]:
+ created = _row_field(row, "created_at", index=5)
+ updated = _row_field(row, "updated_at", index=6)
+ layout = _row_field(row, "layout_json", index=3) or {}
+ return {
+ "id": int(_row_field(row, "id", index=0)),
+ "propertyId": int(_row_field(row, "property_id", index=1)),
+ "name": _row_field(row, "name", index=2),
+ "layoutJson": layout,
+ "isDefault": bool(_row_field(row, "is_default", index=4)),
+ "createdAt": created.isoformat() if hasattr(created, "isoformat") else str(created or ""),
+ "updatedAt": updated.isoformat() if hasattr(updated, "isoformat") else str(updated or ""),
+ }
+
+
+def list_dashboards(conn: Connection, property_id: int) -> list[dict[str, Any]]:
+ cur = conn.execute(
+ f"{_SELECT} WHERE property_id = %s ORDER BY updated_at DESC",
+ (property_id,),
+ )
+ return [_map_dashboard(row) for row in cur.fetchall() or []]
+
+
+def get_dashboard(conn: Connection, dashboard_id: int, property_id: int) -> dict[str, Any] | None:
+ cur = conn.execute(
+ f"{_SELECT} WHERE id = %s AND property_id = %s",
+ (dashboard_id, property_id),
+ )
+ row = cur.fetchone()
+ return _map_dashboard(row) if row else None
+
+
+def create_dashboard(
+ conn: Connection,
+ property_id: int,
+ name: str,
+ layout_json: Any,
+) -> dict[str, Any]:
+ cur = conn.execute(
+ """
+ INSERT INTO dashboards (property_id, name, layout_json)
+ VALUES (%s, %s, %s)
+ RETURNING id, property_id, name, layout_json, is_default, created_at, updated_at
+ """,
+ (property_id, name, Json(layout_json)),
+ )
+ row = cur.fetchone()
+ conn.commit()
+ return _map_dashboard(row)
+
+
+def update_dashboard(
+ conn: Connection,
+ dashboard_id: int,
+ property_id: int,
+ *,
+ name: str | None = None,
+ layout_json: Any | None = None,
+ is_default: bool | None = None,
+) -> dict[str, Any] | None:
+ sets = ["updated_at = now()"]
+ vals: list[Any] = []
+
+ if name is not None:
+ sets.append("name = %s")
+ vals.append(name.strip() or "Untitled dashboard")
+ if layout_json is not None:
+ sets.append("layout_json = %s")
+ vals.append(Json(layout_json))
+ if is_default is not None:
+ if is_default:
+ conn.execute(
+ "UPDATE dashboards SET is_default = false WHERE property_id = %s",
+ (property_id,),
+ )
+ sets.append("is_default = %s")
+ vals.append(is_default)
+
+ vals.extend([dashboard_id, property_id])
+ cur = conn.execute(
+ f"""
+ UPDATE dashboards SET {', '.join(sets)}
+ WHERE id = %s AND property_id = %s
+ RETURNING id, property_id, name, layout_json, is_default, created_at, updated_at
+ """,
+ vals,
+ )
+ row = cur.fetchone()
+ conn.commit()
+ return _map_dashboard(row) if row else None
+
+
+def delete_dashboard(conn: Connection, dashboard_id: int, property_id: int) -> bool:
+ cur = conn.execute(
+ "DELETE FROM dashboards WHERE id = %s AND property_id = %s RETURNING id",
+ (dashboard_id, property_id),
+ )
+ deleted = cur.fetchone() is not None
+ conn.commit()
+ return deleted
diff --git a/src/website_profiling/db/issue_status_store.py b/src/website_profiling/db/issue_status_store.py
new file mode 100644
index 00000000..05870569
--- /dev/null
+++ b/src/website_profiling/db/issue_status_store.py
@@ -0,0 +1,100 @@
+"""Issue workflow status persistence (issue_status table)."""
+from __future__ import annotations
+
+import hashlib
+from typing import Any, Optional
+
+from psycopg import Connection
+
+from ._common import _row_field
+
+_VALID_STATUS = frozenset({"open", "in_progress", "fixed", "ignored"})
+
+_SELECT_COLUMNS = """
+ id, property_id, report_id, issue_fingerprint, category_id,
+ message, url, priority, status, assignee, note, updated_at
+"""
+
+
+def issue_fingerprint(message: str, url: str, category_id: Optional[str] = None) -> str:
+ raw = f"{category_id or ''}|{url or ''}|{message or ''}"
+ return hashlib.sha256(raw.encode()).hexdigest()[:32]
+
+
+def _map_issue_row(row: Any) -> dict[str, Any]:
+ report_id = _row_field(row, "report_id")
+ updated = _row_field(row, "updated_at")
+ return {
+ "id": int(_row_field(row, "id")),
+ "propertyId": int(_row_field(row, "property_id")),
+ "reportId": int(report_id) if report_id is not None else None,
+ "issueFingerprint": _row_field(row, "issue_fingerprint"),
+ "categoryId": _row_field(row, "category_id"),
+ "message": _row_field(row, "message"),
+ "url": _row_field(row, "url"),
+ "priority": _row_field(row, "priority"),
+ "status": _row_field(row, "status"),
+ "assignee": _row_field(row, "assignee"),
+ "note": _row_field(row, "note"),
+ "updatedAt": updated.isoformat() if hasattr(updated, "isoformat") else str(updated or ""),
+ }
+
+
+def list_issue_status(conn: Connection, property_id: int) -> list[dict[str, Any]]:
+ cur = conn.execute(
+ f"""SELECT {_SELECT_COLUMNS}
+ FROM issue_status
+ WHERE property_id = %s
+ ORDER BY updated_at DESC""",
+ (property_id,),
+ )
+ return [_map_issue_row(row) for row in cur.fetchall() or []]
+
+
+def upsert_issue_status(
+ conn: Connection,
+ *,
+ property_id: int,
+ message: str,
+ status: str,
+ report_id: int | None = None,
+ url: str = "",
+ priority: str = "Medium",
+ category_id: str | None = None,
+ assignee: str | None = None,
+ note: str | None = None,
+) -> dict[str, Any]:
+ if status not in _VALID_STATUS:
+ raise ValueError(f"invalid status: {status}")
+
+ fp = issue_fingerprint(message, url, category_id)
+ cur = conn.execute(
+ f"""INSERT INTO issue_status
+ (property_id, report_id, issue_fingerprint, category_id, message, url,
+ priority, status, assignee, note, updated_at)
+ VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, now())
+ ON CONFLICT (property_id, issue_fingerprint) DO UPDATE SET
+ status = EXCLUDED.status,
+ assignee = COALESCE(EXCLUDED.assignee, issue_status.assignee),
+ note = COALESCE(EXCLUDED.note, issue_status.note),
+ report_id = COALESCE(EXCLUDED.report_id, issue_status.report_id),
+ updated_at = now()
+ RETURNING {_SELECT_COLUMNS}""",
+ (
+ property_id,
+ report_id,
+ fp,
+ category_id,
+ message,
+ url,
+ priority,
+ status,
+ assignee,
+ note,
+ ),
+ )
+ row = cur.fetchone()
+ conn.commit()
+ if not row:
+ raise RuntimeError("issue status upsert failed")
+ return _map_issue_row(row)
diff --git a/src/website_profiling/db/markdown_store.py b/src/website_profiling/db/markdown_store.py
index 9dbbcd28..0677e62c 100644
--- a/src/website_profiling/db/markdown_store.py
+++ b/src/website_profiling/db/markdown_store.py
@@ -5,7 +5,7 @@
from psycopg import Connection
-from ._common import _executemany, _now_iso
+from ._common import _executemany, _now_iso, _row_field
_MD_BATCH_SIZE = 200
@@ -71,7 +71,15 @@ def read_page_markdown(conn: Connection, crawl_run_id: int, url: str) -> Optiona
row = cur.fetchone()
if row is None:
return None
- return dict(row)
+ return {
+ "url": _row_field(row, "url"),
+ "title": _row_field(row, "title"),
+ "markdown": _row_field(row, "markdown"),
+ "word_count": _row_field(row, "word_count"),
+ "strategy": _row_field(row, "strategy"),
+ "source_byte_length": _row_field(row, "source_byte_length"),
+ "extracted_at": _row_field(row, "extracted_at"),
+ }
except Exception:
return None
@@ -97,7 +105,7 @@ def list_page_markdown(
(crawl_run_id, pattern),
)
total_row = count_cur.fetchone()
- total = int(dict(total_row).get("count", 0)) if total_row else 0
+ total = int(_row_field(total_row, "count", index=0) or 0) if total_row else 0
cur = conn.execute(
"""SELECT url, title, word_count, strategy, extracted_at
@@ -113,7 +121,7 @@ def list_page_markdown(
(crawl_run_id,),
)
total_row = count_cur.fetchone()
- total = int(dict(total_row).get("count", 0)) if total_row else 0
+ total = int(_row_field(total_row, "count", index=0) or 0) if total_row else 0
cur = conn.execute(
"""SELECT url, title, word_count, strategy, extracted_at
@@ -123,7 +131,16 @@ def list_page_markdown(
LIMIT %s OFFSET %s""",
(crawl_run_id, limit, offset),
)
- items = [dict(row) for row in cur.fetchall()]
+ items = [
+ {
+ "url": _row_field(row, "url"),
+ "title": _row_field(row, "title"),
+ "word_count": _row_field(row, "word_count"),
+ "strategy": _row_field(row, "strategy"),
+ "extracted_at": _row_field(row, "extracted_at"),
+ }
+ for row in cur.fetchall() or []
+ ]
return {"items": items, "total": total, "limit": limit, "offset": offset}
except Exception:
return {"items": [], "total": 0, "limit": limit, "offset": offset}
@@ -141,7 +158,10 @@ def count_page_markdown_by_run(conn: Connection, crawl_run_ids: list[int]) -> di
GROUP BY crawl_run_id""",
(crawl_run_ids,),
)
- return {int(row["crawl_run_id"]): int(row["cnt"]) for row in cur.fetchall()}
+ return {
+ int(_row_field(row, "crawl_run_id")): int(_row_field(row, "cnt") or 0)
+ for row in cur.fetchall() or []
+ }
except Exception:
return {}
@@ -159,3 +179,47 @@ def delete_page_markdown_for_run(conn: Connection, crawl_run_id: int, *, commit:
return deleted
except Exception:
return 0
+
+
+def list_markdown_crawl_runs(
+ conn: Connection,
+ property_id: int | None = None,
+ *,
+ limit: int = 50,
+) -> list[dict[str, Any]]:
+ """Crawl runs with HTML and markdown page counts for the page-markdown UI."""
+ limit = max(1, min(int(limit), 100))
+ where = "WHERE cr.property_id = %s" if property_id else ""
+ params: tuple[Any, ...] = (property_id, limit) if property_id else (limit,)
+ cur = conn.execute(
+ f"""
+ SELECT cr.id, cr.created_at, cr.start_url,
+ COALESCE(html_counts.cnt, 0) AS html_page_count,
+ COALESCE(md_counts.cnt, 0) AS markdown_page_count
+ FROM crawl_runs cr
+ LEFT JOIN (
+ SELECT crawl_run_id, COUNT(*)::int AS cnt
+ FROM crawl_page_html GROUP BY crawl_run_id
+ ) html_counts ON html_counts.crawl_run_id = cr.id
+ LEFT JOIN (
+ SELECT crawl_run_id, COUNT(*)::int AS cnt
+ FROM crawl_page_markdown GROUP BY crawl_run_id
+ ) md_counts ON md_counts.crawl_run_id = cr.id
+ {where}
+ ORDER BY cr.id DESC
+ LIMIT %s
+ """,
+ params,
+ )
+ runs: list[dict[str, Any]] = []
+ for row in cur.fetchall() or []:
+ created = _row_field(row, "created_at")
+ runs.append({
+ "id": int(_row_field(row, "id")),
+ "created_at": created.isoformat() if hasattr(created, "isoformat") else str(created or "") or None,
+ "start_url": _row_field(row, "start_url"),
+ "html_page_count": int(_row_field(row, "html_page_count") or 0),
+ "markdown_page_count": int(_row_field(row, "markdown_page_count") or 0),
+ })
+ return runs
+
diff --git a/src/website_profiling/db/pipeline_jobs.py b/src/website_profiling/db/pipeline_jobs.py
new file mode 100644
index 00000000..0eb85f2d
--- /dev/null
+++ b/src/website_profiling/db/pipeline_jobs.py
@@ -0,0 +1,262 @@
+"""Pipeline job DB helpers — shared by FastAPI routers and the worker process."""
+from __future__ import annotations
+
+import os
+from typing import Any, Optional
+
+from psycopg import Connection
+
+from .pool import db_session
+
+# Stale job thresholds (minutes for pending, hours for running)
+_STALE_PENDING_MINUTES = int(os.getenv("PIPELINE_JOB_STALE_PENDING_MINUTES", "10"))
+_STALE_RUNNING_HOURS = int(os.getenv("PIPELINE_JOB_STALE_HOURS", "1"))
+
+PIPELINE_LOG_MAX = 256_000
+PIPELINE_LOG_TRIM = 200_000
+
+
+def _trim_log(existing: str, chunk: str) -> tuple[str, bool]:
+ combined = existing + chunk
+ if len(combined) <= PIPELINE_LOG_MAX:
+ return combined, False
+ return combined[-PIPELINE_LOG_TRIM:], True
+
+
+# ── Enqueue ──────────────────────────────────────────────────────────────────
+
+def enqueue_job(
+ conn: Connection,
+ job_id: str,
+ job_type: str,
+ command: Optional[str],
+ property_id: Optional[int],
+ config_hash: Optional[str] = None,
+) -> bool:
+ """INSERT a pending job. Returns True if inserted, False if a job is already pending/running."""
+ reconcile_stale_jobs(conn)
+ cur = conn.execute(
+ """INSERT INTO pipeline_jobs (id, job_type, status, command, property_id, config_hash)
+ SELECT %s::uuid, %s, 'pending', %s, %s, %s
+ WHERE NOT EXISTS (
+ SELECT 1 FROM pipeline_jobs WHERE status IN ('pending', 'running')
+ )
+ RETURNING id""",
+ (job_id, job_type, command, property_id, config_hash),
+ )
+ conn.commit()
+ return cur.fetchone() is not None
+
+
+# ── Worker claim ─────────────────────────────────────────────────────────────
+
+def try_claim_pending_job(conn: Connection, worker_pid: int) -> Optional[dict[str, Any]]:
+ """Atomically claim one pending job for the worker. Returns the job row or None."""
+ cur = conn.execute(
+ """UPDATE pipeline_jobs
+ SET status = 'running', worker_pid = %s
+ WHERE id = (
+ SELECT id FROM pipeline_jobs
+ WHERE status = 'pending'
+ ORDER BY started_at ASC
+ LIMIT 1
+ FOR UPDATE SKIP LOCKED
+ )
+ RETURNING id, job_type, command, property_id""",
+ (worker_pid,),
+ )
+ row = cur.fetchone()
+ conn.commit()
+ if row is None:
+ return None
+ return {
+ "id": str(row["id"]),
+ "job_type": str(row["job_type"]),
+ "command": row["command"],
+ "property_id": row["property_id"],
+ }
+
+
+# ── Log appending ────────────────────────────────────────────────────────────
+
+def append_job_log(conn: Connection, job_id: str, chunk: str) -> bool:
+ """Append to log_text with row-level lock. Returns True if log was truncated."""
+ cur = conn.execute(
+ "SELECT log_text, log_truncated FROM pipeline_jobs WHERE id = %s::uuid FOR UPDATE",
+ (job_id,),
+ )
+ row = cur.fetchone()
+ if not row:
+ conn.rollback()
+ return False
+ log, truncated = _trim_log(str(row["log_text"] or ""), chunk)
+ log_truncated = bool(row["log_truncated"]) or truncated
+ conn.execute(
+ "UPDATE pipeline_jobs SET log_text = %s, log_truncated = %s WHERE id = %s::uuid",
+ (log, log_truncated, job_id),
+ )
+ conn.commit()
+ return log_truncated
+
+
+# ── Finish ───────────────────────────────────────────────────────────────────
+
+def finish_job(
+ conn: Connection,
+ job_id: str,
+ status: str,
+ exit_code: Optional[int],
+ error: Optional[str] = None,
+ log_truncated: Optional[bool] = None,
+) -> None:
+ if log_truncated is None:
+ conn.execute(
+ """UPDATE pipeline_jobs
+ SET status = %s, exit_code = %s, error_text = %s, finished_at = now(), worker_pid = NULL
+ WHERE id = %s::uuid""",
+ (status, exit_code, error, job_id),
+ )
+ else:
+ conn.execute(
+ """UPDATE pipeline_jobs
+ SET status = %s, exit_code = %s, error_text = %s, finished_at = now(),
+ log_truncated = %s, worker_pid = NULL
+ WHERE id = %s::uuid""",
+ (status, exit_code, error, log_truncated, job_id),
+ )
+ conn.commit()
+
+
+# ── Flags ────────────────────────────────────────────────────────────────────
+
+def check_flags(conn: Connection, job_id: str) -> tuple[bool, bool]:
+ """Return (cancel_requested, pause_requested) for a running job."""
+ cur = conn.execute(
+ "SELECT cancel_requested, pause_requested FROM pipeline_jobs WHERE id = %s::uuid",
+ (job_id,),
+ )
+ row = cur.fetchone()
+ if not row:
+ return False, False
+ return bool(row["cancel_requested"]), bool(row["pause_requested"])
+
+
+def set_cancel_flag(conn: Connection, job_id: str) -> bool:
+ cur = conn.execute(
+ """UPDATE pipeline_jobs SET cancel_requested = true
+ WHERE id = %s::uuid AND status = 'running'
+ RETURNING id""",
+ (job_id,),
+ )
+ conn.commit()
+ return cur.fetchone() is not None
+
+
+def set_pause_flag(conn: Connection, job_id: str) -> bool:
+ cur = conn.execute(
+ """UPDATE pipeline_jobs SET pause_requested = true
+ WHERE id = %s::uuid AND status = 'running'
+ RETURNING id""",
+ (job_id,),
+ )
+ conn.commit()
+ return cur.fetchone() is not None
+
+
+# ── Reconcile stale jobs ─────────────────────────────────────────────────────
+
+def reconcile_stale_jobs(conn: Connection) -> int:
+ """Mark stale running/pending jobs as error. Returns count reconciled."""
+ cur = conn.execute(
+ """UPDATE pipeline_jobs
+ SET status = 'error',
+ error_text = COALESCE(error_text, 'Job interrupted (server restart or timeout)'),
+ finished_at = now()
+ WHERE status = 'running'
+ AND started_at < now() - (%s::text || ' hours')::interval
+ RETURNING id""",
+ (str(_STALE_RUNNING_HOURS),),
+ )
+ count = len(cur.fetchall())
+
+ cur2 = conn.execute(
+ """UPDATE pipeline_jobs
+ SET status = 'error',
+ error_text = 'Job never started (worker restart)',
+ finished_at = now()
+ WHERE status = 'pending'
+ AND started_at < now() - (%s::text || ' minutes')::interval
+ RETURNING id""",
+ (str(_STALE_PENDING_MINUTES),),
+ )
+ count += len(cur2.fetchall())
+ if count:
+ conn.commit()
+ return count
+
+
+# ── Read helpers ─────────────────────────────────────────────────────────────
+
+def get_job(conn: Connection, job_id: str) -> Optional[dict[str, Any]]:
+ cur = conn.execute(
+ """SELECT id, job_type, status, exit_code, log_text, error_text,
+ log_truncated, property_id, started_at, finished_at, command
+ FROM pipeline_jobs WHERE id = %s::uuid""",
+ (job_id,),
+ )
+ row = cur.fetchone()
+ if not row:
+ return None
+ return _job_row_to_dict(row)
+
+
+def list_jobs(conn: Connection, limit: int = 50) -> list[dict[str, Any]]:
+ reconcile_stale_jobs(conn)
+ cur = conn.execute(
+ """SELECT id, job_type, status, exit_code, log_text, error_text,
+ log_truncated, property_id, started_at, finished_at, command
+ FROM pipeline_jobs ORDER BY started_at DESC LIMIT %s""",
+ (limit,),
+ )
+ return [_job_row_to_dict(r) for r in cur.fetchall()]
+
+
+def get_active_job(conn: Connection) -> Optional[dict[str, Any]]:
+ cur = conn.execute(
+ """SELECT id, job_type, status, exit_code, log_text, error_text,
+ log_truncated, property_id, started_at, finished_at, command
+ FROM pipeline_jobs WHERE status IN ('pending', 'running')
+ ORDER BY started_at DESC LIMIT 1""",
+ )
+ row = cur.fetchone()
+ return _job_row_to_dict(row) if row else None
+
+
+def cancel_job_in_db(conn: Connection, job_id: str, message: str = "Cancelled by user") -> bool:
+ cur = conn.execute(
+ """UPDATE pipeline_jobs
+ SET status = 'error', error_text = %s, exit_code = -1, finished_at = now()
+ WHERE id = %s::uuid AND status IN ('pending', 'running')
+ RETURNING id""",
+ (message, job_id),
+ )
+ conn.commit()
+ return cur.fetchone() is not None
+
+
+def _job_row_to_dict(row: Any) -> dict[str, Any]:
+ started_at = row["started_at"]
+ finished_at = row["finished_at"]
+ return {
+ "id": str(row["id"]),
+ "jobType": str(row["job_type"] or ""),
+ "status": str(row["status"] or ""),
+ "exitCode": row["exit_code"],
+ "log": str(row["log_text"] or ""),
+ "error": row["error_text"],
+ "logTruncated": bool(row["log_truncated"]),
+ "propertyId": row["property_id"],
+ "startedAt": started_at.isoformat() if started_at else None,
+ "finishedAt": finished_at.isoformat() if finished_at else None,
+ "command": row["command"],
+ }
diff --git a/src/website_profiling/db/portfolio_store.py b/src/website_profiling/db/portfolio_store.py
new file mode 100644
index 00000000..ad89d364
--- /dev/null
+++ b/src/website_profiling/db/portfolio_store.py
@@ -0,0 +1,38 @@
+"""Portfolio item deletion (report_payload / crawl_runs)."""
+from __future__ import annotations
+
+from psycopg import Connection
+
+
+def delete_portfolio_report(conn: Connection, report_id: int) -> bool:
+ cur = conn.execute(
+ "DELETE FROM report_payload WHERE id = %s RETURNING id",
+ (report_id,),
+ )
+ deleted = cur.fetchone() is not None
+ conn.commit()
+ return deleted
+
+
+def delete_portfolio_crawl_run(conn: Connection, crawl_run_id: int) -> bool:
+ cur = conn.execute(
+ "DELETE FROM crawl_runs WHERE id = %s RETURNING id",
+ (crawl_run_id,),
+ )
+ deleted = cur.fetchone() is not None
+ conn.commit()
+ return deleted
+
+
+def delete_portfolio_item(
+ conn: Connection,
+ *,
+ report_id: int | None = None,
+ crawl_run_id: int | None = None,
+) -> bool:
+ deleted = False
+ if report_id is not None:
+ deleted = delete_portfolio_report(conn, report_id)
+ if crawl_run_id is not None:
+ deleted = delete_portfolio_crawl_run(conn, crawl_run_id)
+ return deleted
diff --git a/src/website_profiling/db/property_store.py b/src/website_profiling/db/property_store.py
index 434a2a76..85da80a0 100644
--- a/src/website_profiling/db/property_store.py
+++ b/src/website_profiling/db/property_store.py
@@ -206,3 +206,219 @@ def list_properties_public(conn: Connection) -> list[dict[str, Any]]:
"crawl_authorized_at": crawl_auth.isoformat() if crawl_auth else None,
})
return out
+
+
+def get_property_id_by_domain(conn: Connection, domain: str) -> int | None:
+ """Resolve property id from canonical domain (case-insensitive)."""
+ normalized = (domain or "").strip().lower()
+ if not normalized:
+ return None
+ prop = get_property_by_domain(conn, normalized)
+ return int(prop["id"]) if prop else None
+
+
+def resolve_property_id_for_page(
+ conn: Connection,
+ page_url: str,
+ property_id_str: str | None = None,
+ domain_str: str | None = None,
+) -> int | None:
+ """Resolve property ID from explicit param, domain, or URL hostname."""
+ if property_id_str:
+ try:
+ return int(property_id_str)
+ except (ValueError, TypeError):
+ pass
+
+ if domain_str:
+ prop_id = get_property_id_by_domain(conn, domain_str)
+ if prop_id is not None:
+ return prop_id
+
+ host = _extract_hostname(page_url)
+ if host:
+ return get_property_id_by_domain(conn, host)
+ return None
+
+
+def get_property_ops(conn: Connection, property_id: int) -> dict[str, Any] | None:
+ cur = conn.execute(
+ "SELECT schedule_cron, alert_webhook_url, alert_email FROM properties WHERE id = %s",
+ (property_id,),
+ )
+ row = cur.fetchone()
+ if not row:
+ return None
+ return {
+ "schedule_cron": _row_field(row, "schedule_cron", index=0),
+ "alert_webhook_url": _row_field(row, "alert_webhook_url", index=1),
+ "alert_email": _row_field(row, "alert_email", index=2),
+ }
+
+
+def update_property_ops(
+ conn: Connection,
+ property_id: int,
+ *,
+ schedule_cron: str | None,
+ alert_webhook_url: str | None,
+ alert_email: str | None,
+) -> None:
+ conn.execute(
+ """
+ UPDATE properties
+ SET schedule_cron = %s,
+ alert_webhook_url = %s,
+ alert_email = %s,
+ updated_at = now()
+ WHERE id = %s
+ """,
+ (schedule_cron, alert_webhook_url, alert_email, property_id),
+ )
+ conn.commit()
+
+
+def delete_property(conn: Connection, property_id: int) -> bool:
+ cur = conn.execute(
+ "DELETE FROM properties WHERE id = %s RETURNING id",
+ (property_id,),
+ )
+ deleted = cur.fetchone() is not None
+ conn.commit()
+ return deleted
+
+
+def update_property_crawl_preset(
+ conn: Connection,
+ property_id: int,
+ preset: str | None,
+) -> None:
+ conn.execute(
+ "UPDATE properties SET default_crawl_preset = %s, updated_at = now() WHERE id = %s",
+ (preset, property_id),
+ )
+ conn.commit()
+
+
+def authorize_property_crawl(conn: Connection, property_id: int) -> None:
+ """Mark property as crawl-authorized (OAuth flow)."""
+ conn.execute(
+ "UPDATE properties SET crawl_authorized_at = now(), updated_at = now() WHERE id = %s",
+ (property_id,),
+ )
+ conn.commit()
+
+
+def get_property_google_public_status(conn: Connection, property_id: int) -> dict[str, Any]:
+ row = get_property_by_id(conn, property_id)
+ if not row:
+ return {
+ "connected": False,
+ "authMode": None,
+ "gscSiteUrl": None,
+ "ga4PropertyId": None,
+ "dateRangeDays": 28,
+ "connectedEmail": None,
+ "connectedAt": None,
+ }
+ connected_at = row.get("google_connected_at")
+ return {
+ "connected": connected_at is not None,
+ "authMode": row.get("google_auth_mode"),
+ "gscSiteUrl": row.get("gsc_site_url"),
+ "ga4PropertyId": row.get("ga4_property_id"),
+ "dateRangeDays": int(row.get("google_date_range_days") or 0) or 28,
+ "connectedEmail": row.get("google_connected_email"),
+ "connectedAt": connected_at,
+ }
+
+
+def apply_property_google_credentials_patch(
+ conn: Connection,
+ property_id: int,
+ *,
+ refresh_token: str | None = None,
+ auth_mode: str | None = None,
+ gsc_site_url: str | None = None,
+ ga4_property_id: str | None = None,
+ date_range_days: int | None = None,
+ connected_email: str | None = None,
+ fields_set: frozenset[str] | None = None,
+) -> None:
+ """Merge Google OAuth / site mapping fields on a property row."""
+ allowed = fields_set or frozenset({
+ "refresh_token", "auth_mode", "gsc_site_url", "ga4_property_id",
+ "date_range_days", "connected_email",
+ })
+ sets: list[str] = ["updated_at = now()"]
+ vals: list[Any] = []
+
+ def _add(col: str, val: Any) -> None:
+ sets.append(f"{col} = %s")
+ vals.append(val)
+
+ if "gsc_site_url" in allowed and gsc_site_url is not None:
+ _add("gsc_site_url", gsc_site_url.strip() or None)
+ if "ga4_property_id" in allowed and ga4_property_id is not None:
+ v = ga4_property_id.strip() if ga4_property_id else ""
+ if v and not v.isdigit():
+ raise ValueError(
+ "Analytics property ID must be a numeric ID (e.g. 123456789). "
+ "The G-XXXXXXX code is a Measurement ID."
+ )
+ _add("ga4_property_id", v or None)
+ if "date_range_days" in allowed and date_range_days is not None and date_range_days > 0:
+ _add("google_date_range_days", date_range_days)
+ if "auth_mode" in allowed and auth_mode is not None:
+ _add("google_auth_mode", auth_mode or None)
+ if "connected_email" in allowed and connected_email is not None:
+ _add("google_connected_email", connected_email.strip() or None)
+ if "refresh_token" in allowed and refresh_token is not None:
+ token = refresh_token.strip()
+ _add("google_refresh_token", token or None)
+ if token:
+ sets.append("google_connected_at = now()")
+ else:
+ sets.append("google_connected_at = NULL")
+ if "connected_email" not in allowed or connected_email is None:
+ sets.append("google_connected_email = NULL")
+
+ if len(vals) == 0:
+ raise ValueError("No valid fields provided")
+
+ vals.append(property_id)
+ conn.execute(
+ f"UPDATE properties SET {', '.join(sets)} WHERE id = %s",
+ vals,
+ )
+ conn.commit()
+
+
+def disconnect_property_google(conn: Connection, property_id: int) -> None:
+ apply_property_google_credentials_patch(
+ conn,
+ property_id,
+ refresh_token="",
+ auth_mode=None,
+ fields_set=frozenset({"refresh_token", "auth_mode"}),
+ )
+
+
+def get_property_google_status(conn: Connection, property_id: int) -> dict[str, Any] | None:
+ """Property-level Google integration status for the integrations UI."""
+ from website_profiling.db.google_app_store import read_google_app_settings
+ from website_profiling.integrations.google.store import read_last_google_fetched_at_for_property
+
+ if not get_property_by_id(conn, property_id):
+ return None
+
+ prop_status = get_property_google_public_status(conn, property_id)
+ app_cfg = read_google_app_settings(conn)
+ has_client_id = bool(app_cfg.get("client_id"))
+
+ return {
+ **prop_status,
+ "hasClientId": has_client_id,
+ "lastFetchedAt": read_last_google_fetched_at_for_property(conn, property_id),
+ "propertyId": property_id,
+ }
diff --git a/src/website_profiling/db/saved_filter_store.py b/src/website_profiling/db/saved_filter_store.py
new file mode 100644
index 00000000..fc7ef4ae
--- /dev/null
+++ b/src/website_profiling/db/saved_filter_store.py
@@ -0,0 +1,59 @@
+"""Saved crawl filters (saved_crawl_filters table)."""
+from __future__ import annotations
+
+from typing import Any
+
+from psycopg import Connection
+from psycopg.types.json import Json
+
+from ._common import _row_field
+
+
+def _map_filter_row(row: Any) -> dict[str, Any]:
+ created = _row_field(row, "created_at")
+ return {
+ "id": _row_field(row, "id"),
+ "propertyId": _row_field(row, "property_id"),
+ "name": _row_field(row, "name"),
+ "filterJson": _row_field(row, "filter_json") or {},
+ "createdAt": created.isoformat() if hasattr(created, "isoformat") else str(created or ""),
+ }
+
+
+def list_saved_filters(conn: Connection, property_id: int) -> list[dict[str, Any]]:
+ cur = conn.execute(
+ """
+ SELECT id, property_id, name, filter_json, created_at
+ FROM saved_crawl_filters
+ WHERE property_id = %s
+ ORDER BY name
+ """,
+ (property_id,),
+ )
+ return [_map_filter_row(row) for row in cur.fetchall() or []]
+
+
+def upsert_saved_filter(
+ conn: Connection,
+ property_id: int,
+ name: str,
+ filter_json: dict[str, Any],
+) -> None:
+ conn.execute(
+ """
+ INSERT INTO saved_crawl_filters (property_id, name, filter_json)
+ VALUES (%s, %s, %s)
+ ON CONFLICT (property_id, name) DO UPDATE SET filter_json = EXCLUDED.filter_json
+ """,
+ (property_id, name, Json(filter_json)),
+ )
+ conn.commit()
+
+
+def delete_saved_filter(conn: Connection, property_id: int, name: str) -> bool:
+ cur = conn.execute(
+ "DELETE FROM saved_crawl_filters WHERE property_id = %s AND name = %s",
+ (property_id, name),
+ )
+ conn.commit()
+ return cur.rowcount > 0
diff --git a/src/website_profiling/integrations/google/gsc_links_store.py b/src/website_profiling/integrations/google/gsc_links_store.py
index 85af6030..d9ca1bd3 100644
--- a/src/website_profiling/integrations/google/gsc_links_store.py
+++ b/src/website_profiling/integrations/google/gsc_links_store.py
@@ -164,3 +164,34 @@ def read_gsc_links_status(
"sampleLinkCount": len(data.get("sample_links") or []),
"latestLinkCount": len(data.get("latest_links") or []),
}
+
+
+def list_backlinks_velocity(
+ conn: Connection,
+ property_id: int,
+ *,
+ limit: int = 52,
+) -> list[dict[str, Any]]:
+ """Referring-domain trend snapshots for Backlinks velocity chart."""
+ from ...db._common import _parse_row_json, _row_field
+
+ limit = max(1, min(int(limit), 52))
+ cur = conn.execute(
+ """SELECT fetched_at, referring_domains, top_domains
+ FROM gsc_links_snapshots
+ WHERE property_id = %s
+ ORDER BY fetched_at ASC
+ LIMIT %s""",
+ (property_id, limit),
+ )
+ snapshots: list[dict[str, Any]] = []
+ for row in cur.fetchall() or []:
+ fetched = _row_field(row, "fetched_at", index=0)
+ top_domains = _parse_row_json(row, "top_domains", index=2)
+ snapshots.append({
+ "capturedAt": fetched.isoformat() if hasattr(fetched, "isoformat") else str(fetched or "") or None,
+ "referringDomains": int(_row_field(row, "referring_domains", index=1) or 0),
+ "topDomains": top_domains if isinstance(top_domains, list) else [],
+ })
+ return snapshots
+
diff --git a/src/website_profiling/integrations/google/keyword_store.py b/src/website_profiling/integrations/google/keyword_store.py
index 6a6b5f4a..8c6460ba 100644
--- a/src/website_profiling/integrations/google/keyword_store.py
+++ b/src/website_profiling/integrations/google/keyword_store.py
@@ -9,7 +9,7 @@
from psycopg import Connection
from psycopg.types.json import Json
-from ...db.storage import _parse_row_json, _sanitize_for_json
+from ...db._common import _parse_row_json, _row_field, _sanitize_for_json
def write_keyword_data(
@@ -118,9 +118,13 @@ def read_keyword_snapshots_for_property(
)
out: list[dict[str, Any]] = []
for row in cur.fetchall():
- data = _parse_row_json(row)
+ data = _parse_row_json(row, "data", index=1)
if isinstance(data, dict):
- out.append({"fetched_at": row["fetched_at"], **data})
+ fetched = _row_field(row, "fetched_at", index=0)
+ out.append({
+ "fetched_at": fetched.isoformat() if hasattr(fetched, "isoformat") else str(fetched or ""),
+ **data,
+ })
return out
except Exception:
return []
@@ -144,15 +148,33 @@ def read_keyword_history(
ORDER BY id DESC LIMIT %s""",
(property_id, keyword, limit),
)
- return [
- {
- "fetched_at": row["fetched_at"],
- "position": row["position"],
- "clicks": row["clicks"],
- "impressions": row["impressions"],
- "ctr": row["ctr"],
- }
- for row in cur.fetchall()
- ]
+ rows = list(cur.fetchall() or [])
+ return [_map_keyword_history_row(row) for row in reversed(rows)]
except Exception:
return []
+
+
+def read_keyword_history_batch(
+ conn: Connection,
+ keywords: list[str],
+ *,
+ property_id: int,
+ limit: int = 30,
+) -> dict[str, list[dict[str, Any]]]:
+ """Batch keyword history keyed by keyword string."""
+ limit = max(1, min(int(limit), 90))
+ results: dict[str, list[dict[str, Any]]] = {}
+ for kw in keywords:
+ results[kw] = read_keyword_history(conn, kw, limit, property_id=property_id)
+ return results
+
+
+def _map_keyword_history_row(row: Any) -> dict[str, Any]:
+ fetched = _row_field(row, "fetched_at", index=0)
+ return {
+ "fetched_at": fetched.isoformat() if hasattr(fetched, "isoformat") else str(fetched or ""),
+ "position": _row_field(row, "position", index=1),
+ "clicks": _row_field(row, "clicks", index=2),
+ "impressions": _row_field(row, "impressions", index=3),
+ "ctr": _row_field(row, "ctr", index=4),
+ }
diff --git a/src/website_profiling/integrations/google/page_snapshot_store.py b/src/website_profiling/integrations/google/page_snapshot_store.py
index 06b4f1a0..3067ac58 100644
--- a/src/website_profiling/integrations/google/page_snapshot_store.py
+++ b/src/website_profiling/integrations/google/page_snapshot_store.py
@@ -7,7 +7,7 @@
from psycopg import Connection
from psycopg.types.json import Json
-from ...db.storage import _parse_row_json, _sanitize_for_json
+from ...db._common import _parse_row_json, _row_field, _sanitize_for_json
from .normalize import normalize_url
from .page_lookup import _public_ga4_page, _public_gsc_page, summary_from_slice
@@ -33,7 +33,7 @@ def write_page_snapshot(conn: Connection, page_url: str, data: dict[str, Any]) -
(page_url.strip(), url_norm, Json(_sanitize_for_json(data))),
)
row = cur.fetchone()
- snapshot_id = int(row["id"]) if row else 0
+ snapshot_id = int(_row_field(row, "id", index=0)) if row else 0
limit = max_snapshots_per_url()
conn.execute(
"""
@@ -60,12 +60,13 @@ def read_page_snapshot(conn: Connection, snapshot_id: int) -> dict[str, Any] | N
row = cur.fetchone()
if not row:
return None
- data = _parse_row_json(row) or {}
+ data = _parse_row_json(row, "data", index=4) or {}
+ fetched = _row_field(row, "fetched_at", index=3)
return {
- "snapshotId": int(row["id"]),
- "pageUrl": str(row["page_url"]),
- "urlNorm": str(row["url_norm"]),
- "fetchedAt": row["fetched_at"].isoformat() if row["fetched_at"] else None,
+ "snapshotId": int(_row_field(row, "id", index=0)),
+ "pageUrl": str(_row_field(row, "page_url", index=1)),
+ "urlNorm": str(_row_field(row, "url_norm", index=2)),
+ "fetchedAt": fetched.isoformat() if hasattr(fetched, "isoformat") else str(fetched or ""),
"source": data.get("source") or "live",
"gsc": data.get("gsc"),
"ga4": data.get("ga4"),
@@ -90,13 +91,14 @@ def list_live_history(
)
out: list[dict[str, Any]] = []
for row in cur.fetchall():
- data = _parse_row_json(row) or {}
+ data = _parse_row_json(row, "data", index=2) or {}
gsc = data.get("gsc")
ga4 = data.get("ga4")
+ fetched = _row_field(row, "fetched_at", index=1)
out.append(
{
- "id": int(row["id"]),
- "fetchedAt": row["fetched_at"].isoformat() if row["fetched_at"] else None,
+ "id": int(_row_field(row, "id", index=0)),
+ "fetchedAt": fetched.isoformat() if hasattr(fetched, "isoformat") else str(fetched or ""),
"type": "live",
**summary_from_slice(gsc, ga4),
}
@@ -111,6 +113,59 @@ def latest_live_snapshot(conn: Connection, page_url: str) -> dict[str, Any] | No
return read_page_snapshot(conn, int(rows[0]["id"]))
+def read_page_snapshot_compare(conn: Connection, snapshot_id: int) -> dict[str, Any] | None:
+ """Load snapshot for page-compare API ({id, fetchedAt, data})."""
+ cur = conn.execute(
+ "SELECT id, fetched_at, data FROM page_google_snapshots WHERE id = %s",
+ (snapshot_id,),
+ )
+ row = cur.fetchone()
+ if not row:
+ return None
+ data = _parse_row_json(row, "data", index=2)
+ if not isinstance(data, dict):
+ data = {}
+ fetched = _row_field(row, "fetched_at", index=1)
+ return {
+ "id": int(_row_field(row, "id", index=0)),
+ "fetchedAt": fetched.isoformat() if hasattr(fetched, "isoformat") else str(fetched or ""),
+ "data": data,
+ }
+
+
+def list_page_snapshot_api_history(
+ conn: Connection,
+ page_url: str,
+ *,
+ limit: int = 15,
+) -> list[dict[str, Any]]:
+ """History rows with raw gsc/ga4 blobs for the integrations API."""
+ url_norm = normalize_url(page_url)
+ cur = conn.execute(
+ """
+ SELECT id, fetched_at, data
+ FROM page_google_snapshots
+ WHERE url_norm = %s
+ ORDER BY fetched_at DESC, id DESC
+ LIMIT %s
+ """,
+ (url_norm, limit),
+ )
+ out: list[dict[str, Any]] = []
+ for row in cur.fetchall() or []:
+ data = _parse_row_json(row, "data", index=2) or {}
+ if not isinstance(data, dict):
+ data = {}
+ fetched = _row_field(row, "fetched_at", index=1)
+ out.append({
+ "id": int(_row_field(row, "id", index=0)),
+ "fetchedAt": fetched.isoformat() if hasattr(fetched, "isoformat") else str(fetched or ""),
+ "gsc": data.get("gsc"),
+ "ga4": data.get("ga4"),
+ })
+ return out
+
+
def package_live_payload(
page_url: str,
gsc: dict[str, Any] | None,
diff --git a/src/website_profiling/integrations/google/store.py b/src/website_profiling/integrations/google/store.py
index ba5fdece..1c0e9334 100644
--- a/src/website_profiling/integrations/google/store.py
+++ b/src/website_profiling/integrations/google/store.py
@@ -11,7 +11,7 @@
from psycopg import Connection
from psycopg.types.json import Json
-from ...db.storage import _parse_row_json, _sanitize_for_json
+from ...db._common import _parse_row_json, _row_field, _sanitize_for_json
def write_google_data(
@@ -128,6 +128,125 @@ def read_prior_google_snapshot(
return None
+def read_last_google_fetched_at(conn: Connection) -> str | None:
+ """ISO timestamp of the most recent google_data row (any property)."""
+ try:
+ cur = conn.execute(
+ "SELECT fetched_at FROM google_data ORDER BY id DESC LIMIT 1"
+ )
+ row = cur.fetchone()
+ if not row:
+ return None
+ fetched = _row_field(row, "fetched_at", index=0)
+ if fetched is None:
+ return None
+ return fetched.isoformat() if hasattr(fetched, "isoformat") else str(fetched)
+ except Exception:
+ return None
+
+
+def read_google_snapshot_row(
+ conn: Connection,
+ property_id: int,
+ *,
+ snapshot_id: int | None = None,
+) -> dict[str, Any] | None:
+ """Return one google_data row as {id, fetchedAt, data} with full parsed blob."""
+ try:
+ if snapshot_id is not None:
+ cur = conn.execute(
+ """
+ SELECT id, fetched_at, data
+ FROM google_data
+ WHERE id = %s AND property_id = %s
+ """,
+ (snapshot_id, property_id),
+ )
+ else:
+ cur = conn.execute(
+ """
+ SELECT id, fetched_at, data
+ FROM google_data
+ WHERE property_id = %s
+ ORDER BY id DESC
+ LIMIT 1
+ """,
+ (property_id,),
+ )
+ row = cur.fetchone()
+ if not row:
+ return None
+ data = _parse_row_json(row, "data", index=2)
+ if not isinstance(data, dict):
+ return None
+ fetched = _row_field(row, "fetched_at", index=1)
+ return {
+ "id": int(_row_field(row, "id", index=0)),
+ "fetchedAt": fetched.isoformat() if hasattr(fetched, "isoformat") else str(fetched or ""),
+ "data": data,
+ }
+ except Exception:
+ return None
+
+
+def list_google_snapshot_rows(
+ conn: Connection,
+ property_id: int,
+ *,
+ limit: int = 10,
+) -> list[dict[str, Any]]:
+ """Recent google_data rows for a property as {id, fetchedAt, data}."""
+ limit = max(1, min(int(limit), 50))
+ try:
+ cur = conn.execute(
+ """
+ SELECT id, fetched_at, data
+ FROM google_data
+ WHERE property_id = %s
+ ORDER BY id DESC
+ LIMIT %s
+ """,
+ (property_id, limit),
+ )
+ out: list[dict[str, Any]] = []
+ for row in cur.fetchall() or []:
+ data = _parse_row_json(row, "data", index=2)
+ if not isinstance(data, dict):
+ continue
+ fetched = _row_field(row, "fetched_at", index=1)
+ out.append({
+ "id": int(_row_field(row, "id", index=0)),
+ "fetchedAt": fetched.isoformat() if hasattr(fetched, "isoformat") else str(fetched or ""),
+ "data": data,
+ })
+ return out
+ except Exception:
+ return []
+
+
+def read_last_google_fetched_at_for_property(conn: Connection, property_id: int) -> str | None:
+ """ISO timestamp of the most recent google_data row for a property."""
+ try:
+ cur = conn.execute(
+ """
+ SELECT fetched_at FROM google_data
+ WHERE property_id = %s
+ ORDER BY id DESC
+ LIMIT 1
+ """,
+ (property_id,),
+ )
+ row = cur.fetchone()
+ if not row:
+ return None
+ fetched = _row_field(row, "fetched_at", index=0)
+ if fetched is None:
+ return None
+ return fetched.isoformat() if hasattr(fetched, "isoformat") else str(fetched)
+ except Exception:
+ return None
+
+
def gsc_row_deltas(
current_rows: list[dict[str, Any]],
prior_rows: list[dict[str, Any]],
diff --git a/src/website_profiling/integrations/keywords/competitor_gap_store.py b/src/website_profiling/integrations/keywords/competitor_gap_store.py
index ea585b2a..7c6ca8b2 100644
--- a/src/website_profiling/integrations/keywords/competitor_gap_store.py
+++ b/src/website_profiling/integrations/keywords/competitor_gap_store.py
@@ -1,7 +1,6 @@
"""Read/write per-property competitor keyword gap rows."""
from __future__ import annotations
-import json
from typing import Any
from psycopg import Connection
@@ -25,7 +24,7 @@ def read_competitor_keyword_gap(conn: Connection, property_id: int | None) -> li
)
row = cur.fetchone()
if row is None:
- return _migrate_legacy_config_if_empty(conn, property_id)
+ return []
data = _parse_row_json(row)
if isinstance(data, list):
return [r for r in data if isinstance(r, dict)]
@@ -34,27 +33,6 @@ def read_competitor_keyword_gap(conn: Connection, property_id: int | None) -> li
return []
-def _migrate_legacy_config_if_empty(conn: Connection, property_id: int) -> list[dict[str, Any]]:
- """One-time read from global pipeline_config when property has no rows yet."""
- try:
- from ...config import get_str
- from ...db.config_store import read_pipeline_config
-
- known, _ = read_pipeline_config(conn)
- raw = (get_str(known or {}, "competitor_keyword_gap_json", "") or "").strip()
- if not raw:
- return []
- parsed = json.loads(raw)
- if not isinstance(parsed, list):
- return []
- rows = [r for r in parsed if isinstance(r, dict)]
- if rows:
- write_competitor_keyword_gap(conn, property_id, rows)
- return rows
- except Exception:
- return []
-
-
def write_competitor_keyword_gap(
conn: Connection,
property_id: int,
diff --git a/src/website_profiling/llm/agent.py b/src/website_profiling/llm/agent.py
index 25d0d0a2..936677d3 100644
--- a/src/website_profiling/llm/agent.py
+++ b/src/website_profiling/llm/agent.py
@@ -77,7 +77,7 @@ def _max_tool_rounds(cfg: dict[str, str]) -> int:
- Export lists: export_list_as_csv with the matching list tool
Export playbook (chat UI shows download buttons after export tools — do not paste file contents):
-- Full audit PDF/HTML/CSV/JSON: export_audit_report with format pdf|html|csv|json
+- Full audit PDF/CSV/JSON: export_audit_report with format pdf|csv|json (PDF via FileService)
- Compare issue diff CSV: export_compare_csv with baseline_report_id
- Export a list as CSV: export_list_as_csv with tool_name and tool_args (e.g. list_broken_links)
- After export tools succeed, tell the user their download is ready; the UI renders file buttons automatically
diff --git a/src/website_profiling/llm/audit_summary.py b/src/website_profiling/llm/audit_summary.py
index 551582df..fc530802 100644
--- a/src/website_profiling/llm/audit_summary.py
+++ b/src/website_profiling/llm/audit_summary.py
@@ -17,7 +17,7 @@ def rank_issues_by_traffic(
for row in gsc_pages or []:
if not isinstance(row, dict):
continue
- url = str(row.get("page") or row.get("url") or "").strip().lower()
+ url = str(row.get("page") or "").strip().lower()
if not url:
continue
try:
@@ -55,7 +55,7 @@ def generate_audit_executive_summary(
categories = report_payload.get("categories") or []
gsc = (report_payload.get("google") or {}).get("gsc") or {}
- gsc_pages = (gsc.get("top_pages") or gsc.get("pages")) if isinstance(gsc, dict) else []
+ gsc_pages = gsc.get("top_pages") if isinstance(gsc, dict) else []
top_issues = rank_issues_by_traffic(categories, gsc_pages)[:5]
scores = [c.get("score") for c in categories if isinstance(c.get("score"), (int, float))]
diff --git a/src/website_profiling/llm/ollama_catalog.py b/src/website_profiling/llm/ollama_catalog.py
new file mode 100644
index 00000000..865a6f8b
--- /dev/null
+++ b/src/website_profiling/llm/ollama_catalog.py
@@ -0,0 +1,174 @@
+"""Ollama local + cloud model catalog (mirrors web/src/server/ollamaModels.ts)."""
+from __future__ import annotations
+
+import json
+import re
+import urllib.error
+import urllib.request
+from typing import Any
+
+OLLAMA_CLOUD_CATALOG_URL = "https://ollama.com/api/tags"
+
+PRO_CLOUD_MODEL_PATTERNS = [
+ re.compile(r"671b", re.I),
+ re.compile(r"480b", re.I),
+ re.compile(r":1t(?:-cloud|:cloud)?$", re.I),
+ re.compile(r"v4-pro", re.I),
+ re.compile(r"nemotron-3-ultra", re.I),
+ re.compile(r"nemotron-3-super", re.I),
+ re.compile(r"mistral-large", re.I),
+ re.compile(r"397b", re.I),
+ re.compile(r"cogito-2\.1:671b", re.I),
+ re.compile(r"deepseek-v4-pro", re.I),
+ re.compile(r"qwen3-coder:480b", re.I),
+ re.compile(r"gpt-oss:120b", re.I),
+]
+
+
+def is_cloud_model_ref(name: str) -> bool:
+ return name.endswith("-cloud") or name.endswith(":cloud")
+
+
+def to_cloud_model_ref(name: str) -> str:
+ trimmed = name.strip()
+ if not trimmed:
+ return trimmed
+ if trimmed.endswith("-cloud") or trimmed.endswith(":cloud"):
+ return trimmed
+ return f"{trimmed}-cloud" if ":" in trimmed else f"{trimmed}:cloud"
+
+
+def resolve_billing_tier(name: str, source: str) -> dict[str, Any]:
+ cloud = source == "cloud" or is_cloud_model_ref(name)
+ if not cloud:
+ return {"billing": "free_local", "requires_subscription": False}
+ if any(p.search(name) for p in PRO_CLOUD_MODEL_PATTERNS):
+ return {"billing": "cloud_pro", "requires_subscription": True}
+ return {"billing": "cloud_free", "requires_subscription": True}
+
+
+def _with_billing(entry: dict[str, Any]) -> dict[str, Any]:
+ tier = resolve_billing_tier(str(entry.get("name") or ""), str(entry.get("source") or "local"))
+ return {**entry, **tier}
+
+
+def _normalize_local_model(raw: dict[str, Any]) -> dict[str, Any] | None:
+ name = str(raw.get("name") or "").strip()
+ if not name:
+ return None
+ cloud = bool(raw.get("remote_host")) or is_cloud_model_ref(name)
+ details = raw.get("details") if isinstance(raw.get("details"), dict) else {}
+ return _with_billing({
+ "name": name,
+ "source": "cloud" if cloud else "local",
+ "installed": True,
+ "capabilities": raw.get("capabilities") if isinstance(raw.get("capabilities"), list) else None,
+ "context_length": details.get("context_length"),
+ })
+
+
+def _normalize_catalog_model(raw: dict[str, Any]) -> dict[str, Any] | None:
+ base = str(raw.get("name") or "").strip()
+ if not base:
+ return None
+ return _with_billing({
+ "name": to_cloud_model_ref(base),
+ "source": "cloud",
+ "installed": False,
+ })
+
+
+def _model_key(name: str) -> str:
+ return name.lower()
+
+
+def merge_ollama_models(
+ local: list[dict[str, Any]],
+ cloud_catalog: list[dict[str, Any]],
+) -> list[dict[str, Any]]:
+ by_key: dict[str, dict[str, Any]] = {}
+ for m in cloud_catalog:
+ by_key[_model_key(str(m.get("name") or ""))] = m
+ for m in local:
+ key = _model_key(str(m.get("name") or ""))
+ existing = by_key.get(key)
+ merged = {
+ **(existing or {}),
+ **m,
+ "installed": True,
+ "capabilities": m.get("capabilities") or (existing or {}).get("capabilities"),
+ "context_length": m.get("context_length") or (existing or {}).get("context_length"),
+ }
+ by_key[key] = _with_billing(merged)
+
+ def sort_key(m: dict[str, Any]) -> tuple:
+ return (
+ 0 if m.get("installed") else 1,
+ 0 if m.get("source") == "local" else 1,
+ str(m.get("name") or ""),
+ )
+
+ return sorted(by_key.values(), key=sort_key)
+
+
+def _fetch_json(url: str, *, timeout: float = 8.0) -> dict[str, Any] | None:
+ try:
+ req = urllib.request.Request(url, headers={"Accept": "application/json"})
+ with urllib.request.urlopen(req, timeout=timeout) as resp:
+ return json.loads(resp.read().decode())
+ except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, OSError):
+ return None
+
+
+def fetch_ollama_models(base_url: str) -> dict[str, Any]:
+ normalized_base = (base_url or "http://127.0.0.1:11434").rstrip("/") or "http://127.0.0.1:11434"
+
+ local_data = _fetch_json(f"{normalized_base}/api/tags", timeout=8.0)
+ cloud_data = _fetch_json(OLLAMA_CLOUD_CATALOG_URL, timeout=12.0)
+
+ local_ok = local_data is not None
+ cloud_catalog_ok = cloud_data is not None
+
+ local_models = [
+ m for raw in (local_data or {}).get("models") or []
+ if isinstance(raw, dict)
+ for m in [_normalize_local_model(raw)]
+ if m is not None
+ ]
+ cloud_models = [
+ m for raw in (cloud_data or {}).get("models") or []
+ if isinstance(raw, dict)
+ for m in [_normalize_catalog_model(raw)]
+ if m is not None
+ ]
+ models = merge_ollama_models(local_models, cloud_models)
+
+ if not local_ok and not cloud_catalog_ok:
+ return {
+ "ok": False,
+ "baseUrl": normalized_base,
+ "models": [],
+ "cloudCatalogOk": False,
+ "localOk": False,
+ "error": "Cannot reach Ollama or the cloud model catalog.",
+ }
+
+ return {
+ "ok": local_ok or cloud_catalog_ok,
+ "baseUrl": normalized_base,
+ "models": models,
+ "cloudCatalogOk": cloud_catalog_ok,
+ "localOk": local_ok,
+ }
+
+
+def model_is_configured(models: list[dict[str, Any]], configured_model: str) -> bool:
+ target = configured_model.strip()
+ if not target:
+ return len(models) > 0
+ key = _model_key(target)
+ return any(_model_key(str(m.get("name") or "")) == key for m in models)
+
+
+def models_support_tools(models: list[dict[str, Any]]) -> bool:
+ return any("tools" in (m.get("capabilities") or []) for m in models)
diff --git a/src/website_profiling/reporting/builder.py b/src/website_profiling/reporting/builder.py
index c4c29ef1..8a547baf 100644
--- a/src/website_profiling/reporting/builder.py
+++ b/src/website_profiling/reporting/builder.py
@@ -685,7 +685,7 @@ def run_simple_report(
gsc_pages = []
gsc_block = (report_data.get("google") or {}).get("gsc") or {}
if isinstance(gsc_block, dict):
- gsc_pages = gsc_block.get("top_pages") or gsc_block.get("pages") or []
+ gsc_pages = gsc_block.get("top_pages") or []
enrich_top_issues_with_llm(
report_data.get("categories") or [],
llm_cfg_for_clusters,
diff --git a/src/website_profiling/reporting/indexation.py b/src/website_profiling/reporting/indexation.py
index 8f8394e6..c6913dfd 100644
--- a/src/website_profiling/reporting/indexation.py
+++ b/src/website_profiling/reporting/indexation.py
@@ -31,12 +31,12 @@ def _gsc_page_urls(google_data: dict[str, Any] | None) -> list[str]:
if not google_data:
return []
gsc = google_data.get("gsc") if isinstance(google_data.get("gsc"), dict) else {}
- raw = gsc.get("top_pages") or gsc.get("pages")
+ raw = gsc.get("top_pages")
pages = raw if isinstance(raw, list) else []
out: list[str] = []
for row in pages:
if isinstance(row, dict):
- u = str(row.get("page") or row.get("url") or "").strip()
+ u = str(row.get("page") or "").strip()
if u:
out.append(u)
return out
@@ -46,12 +46,12 @@ def _gsc_by_page(google_data: dict[str, Any] | None) -> dict[str, dict]:
if not google_data:
return {}
gsc = google_data.get("gsc") if isinstance(google_data.get("gsc"), dict) else {}
- raw = gsc.get("top_pages") or gsc.get("pages")
+ raw = gsc.get("top_pages")
pages = raw if isinstance(raw, list) else []
out: dict[str, dict] = {}
for row in pages:
if isinstance(row, dict):
- u = str(row.get("page") or row.get("url") or "").strip()
+ u = str(row.get("page") or "").strip()
if u:
out[u] = row
return out
diff --git a/src/website_profiling/reporting/issue_impact.py b/src/website_profiling/reporting/issue_impact.py
index fd14f1de..1629e406 100644
--- a/src/website_profiling/reporting/issue_impact.py
+++ b/src/website_profiling/reporting/issue_impact.py
@@ -13,10 +13,10 @@ def _metrics_by_url(google_data: dict[str, Any] | None) -> tuple[dict[str, dict]
return clicks, sessions
gsc = google_data.get("gsc") or {}
if isinstance(gsc, dict):
- for row in (gsc.get("pages") or gsc.get("top_pages") or []):
+ for row in (gsc.get("top_pages") or []):
if not isinstance(row, dict):
continue
- url = str(row.get("page") or row.get("url") or "").strip().lower().rstrip("/")
+ url = str(row.get("page") or "").strip().lower().rstrip("/")
if not url:
continue
clicks[url] = {
@@ -25,10 +25,10 @@ def _metrics_by_url(google_data: dict[str, Any] | None) -> tuple[dict[str, dict]
}
ga4 = google_data.get("ga4") or {}
if isinstance(ga4, dict):
- for row in (ga4.get("pages") or ga4.get("top_pages") or []):
+ for row in (ga4.get("top_pages") or []):
if not isinstance(row, dict):
continue
- path = str(row.get("path") or row.get("pagePath") or row.get("url") or "").strip().lower()
+ path = str(row.get("path") or "").strip().lower()
if not path:
continue
sessions[path] = {"ga4_sessions": float(row.get("sessions") or 0)}
diff --git a/src/website_profiling/reporting/pdf/__init__.py b/src/website_profiling/reporting/pdf/__init__.py
deleted file mode 100644
index c2aabc1a..00000000
--- a/src/website_profiling/reporting/pdf/__init__.py
+++ /dev/null
@@ -1,15 +0,0 @@
-"""PDF document model and export pipeline."""
-from __future__ import annotations
-
-from .builder import build_pdf_document
-from .document import PdfDocument
-from .options import PdfBuildOptions, PdfLimits
-from .render import render_pdf_document
-
-__all__ = [
- "build_pdf_document",
- "render_pdf_document",
- "PdfDocument",
- "PdfBuildOptions",
- "PdfLimits",
-]
diff --git a/src/website_profiling/reporting/pdf/adapters/__init__.py b/src/website_profiling/reporting/pdf/adapters/__init__.py
deleted file mode 100644
index 50c1f7df..00000000
--- a/src/website_profiling/reporting/pdf/adapters/__init__.py
+++ /dev/null
@@ -1,29 +0,0 @@
-"""Section adapter registry.
-
-Each adapter maps a section key to a function that accepts the raw payload
-dict + PdfBuildOptions and returns a list of PdfSection objects. Adapters
-that find no relevant data return an empty list.
-"""
-from __future__ import annotations
-
-from typing import Any, Callable
-
-from ..document import PdfSection
-from ..options import PdfBuildOptions
-
-SectionAdapterFn = Callable[[dict[str, Any], PdfBuildOptions], list[PdfSection]]
-
-# Populated by each sub-module at import time
-SECTION_ADAPTERS: dict[str, SectionAdapterFn] = {}
-
-
-def register(key: str) -> Callable[[SectionAdapterFn], SectionAdapterFn]:
- """Decorator: @register("lighthouse") marks a function as a section adapter."""
- def _wrap(fn: SectionAdapterFn) -> SectionAdapterFn:
- SECTION_ADAPTERS[key] = fn
- return fn
- return _wrap
-
-
-# Import adapters so they self-register
-from . import core, findings, appendix # noqa: E402, F401
diff --git a/src/website_profiling/reporting/pdf/adapters/appendix.py b/src/website_profiling/reporting/pdf/adapters/appendix.py
deleted file mode 100644
index 16329ff5..00000000
--- a/src/website_profiling/reporting/pdf/adapters/appendix.py
+++ /dev/null
@@ -1,68 +0,0 @@
-"""Appendix adapter — crawled URL sample and data-source glossary."""
-from __future__ import annotations
-
-from typing import Any
-
-from ....tools.export_audit_data import _GLOSSARY_ROWS
-from ..document import (
- KeyValueBlock,
- PdfSection,
- PdfTruncation,
- SpacerBlock,
- UrlListBlock,
-)
-from ..options import PdfBuildOptions
-from . import register
-
-
-@register("appendix")
-def adapt_appendix(payload: dict[str, Any], opts: PdfBuildOptions) -> list[PdfSection]:
- if not opts.include_appendix:
- return []
-
- sections: list[PdfSection] = []
-
- # --- Crawled URLs sample ---
- links = [l for l in (payload.get("links") or []) if isinstance(l, dict)]
- if links:
- limit = opts.limits.urls_sample
- sample = links[:limit]
- rows = [
- {
- "url": str(lnk.get("url") or ""),
- "status": str(lnk.get("status") or ""),
- "title": str(lnk.get("title") or "").strip(),
- }
- for lnk in sample
- ]
- has_titles = any(r["title"] for r in rows)
- trunc = PdfTruncation(shown=len(rows), total=len(links)) if len(links) > limit else None
- sections.append(PdfSection(
- id="appendix.urls",
- section_key="links",
- title="Crawled URLs (sample)",
- priority=80,
- page_break_before=False,
- blocks=[
- UrlListBlock(
- id="appendix.url_list",
- rows=rows,
- show_title=has_titles,
- truncation=trunc,
- ),
- SpacerBlock(id="appendix.url_spacer", height_pt=6),
- ],
- ))
-
- # --- Glossary ---
- if opts.include_glossary:
- gloss_rows = [(term, desc) for term, desc in _GLOSSARY_ROWS]
- sections.append(PdfSection(
- id="appendix.glossary",
- section_key="core",
- title="Data source glossary",
- priority=90,
- blocks=[KeyValueBlock(id="appendix.glossary_kv", rows=gloss_rows, layout="glossary")],
- ))
-
- return sections
diff --git a/src/website_profiling/reporting/pdf/adapters/core.py b/src/website_profiling/reporting/pdf/adapters/core.py
deleted file mode 100644
index 27bef7e0..00000000
--- a/src/website_profiling/reporting/pdf/adapters/core.py
+++ /dev/null
@@ -1,38 +0,0 @@
-"""Core adapter — audit-details section (category scores live on cover)."""
-from __future__ import annotations
-
-from typing import Any
-
-from ....tools.export_audit_data import _format_report_date, _summary_lines
-from ..document import KeyValueBlock, PdfSection, SpacerBlock
-from ..options import PdfBuildOptions
-from . import register
-
-
-@register("core")
-def adapt_core(payload: dict[str, Any], opts: PdfBuildOptions) -> list[PdfSection]:
- sections: list[PdfSection] = []
-
- # Category scores are rendered on the cover page — not duplicated here.
-
- # --- Audit details section ---
- summary_rows = _summary_lines(payload)
- if summary_rows:
- formatted_rows: list[tuple[str, str]] = []
- for key, val in summary_rows:
- if key == "Report generated":
- formatted_rows.append((key, _format_report_date(val)))
- else:
- formatted_rows.append((key, val))
- sections.append(PdfSection(
- id="core.audit_details",
- section_key="core",
- title="Audit details",
- priority=70,
- blocks=[
- KeyValueBlock(id="core.audit_kv", rows=formatted_rows, layout="audit"),
- SpacerBlock(id="core.audit_spacer", height_pt=6),
- ],
- ))
-
- return sections
diff --git a/src/website_profiling/reporting/pdf/adapters/findings.py b/src/website_profiling/reporting/pdf/adapters/findings.py
deleted file mode 100644
index ebc18e8d..00000000
--- a/src/website_profiling/reporting/pdf/adapters/findings.py
+++ /dev/null
@@ -1,54 +0,0 @@
-"""Findings adapter — normalizes and groups all audit issues."""
-from __future__ import annotations
-
-from typing import Any
-
-from ....tools.export_audit_data import _issues_rows, _priority_sort_key
-from ..document import PdfSection, PdfTruncation
-from ..normalize import group_issues_for_pdf, normalize_issue_for_pdf
-from ..options import PdfBuildOptions
-from . import register
-
-
-@register("findings")
-def adapt_findings(payload: dict[str, Any], opts: PdfBuildOptions) -> list[PdfSection]:
- raw_rows = _issues_rows(payload)
- if not raw_rows:
- return []
-
- raw_rows = sorted(raw_rows, key=_priority_sort_key)
- total = len(raw_rows)
- capped = raw_rows[: opts.limits.issues_total]
-
- pdf_issues = [
- normalize_issue_for_pdf(row, include_recommendation=opts.include_recommendations)
- for row in capped
- ]
-
- groups = group_issues_for_pdf(
- pdf_issues,
- issues_per_group=opts.limits.issues_per_group,
- issues_total=opts.limits.issues_total,
- )
-
- if not groups:
- return []
-
- section_trunc: PdfTruncation | None = None
- if total > opts.limits.issues_total:
- section_trunc = PdfTruncation(
- shown=opts.limits.issues_total,
- total=total,
- reason="limit",
- continue_in=["CSV", "workbook"],
- )
-
- return [PdfSection(
- id="findings",
- section_key="findings",
- title="Findings",
- priority=20,
- page_break_before=False,
- blocks=list(groups), # type: ignore[arg-type]
- truncation=section_trunc,
- )]
diff --git a/src/website_profiling/reporting/pdf/builder.py b/src/website_profiling/reporting/pdf/builder.py
deleted file mode 100644
index 8f2d30b2..00000000
--- a/src/website_profiling/reporting/pdf/builder.py
+++ /dev/null
@@ -1,178 +0,0 @@
-"""build_pdf_document — assembles a PdfDocument from a raw report payload."""
-from __future__ import annotations
-
-from datetime import datetime, timezone
-from typing import Any, Optional
-
-from ...tools.export_audit_data import (
- _executive_export_data,
- _executive_source_label,
- _format_report_date,
- _issue_priority_counts,
- _issues_rows,
- _overall_score,
- _priority_sort_key,
- _score_band,
-)
-from .document import (
- SCHEMA_VERSION,
- PdfCoverBlock,
- PdfDocument,
- PdfFooterBlock,
- PdfIssue,
- PdfMeta,
- PdfScoreHero,
- ScoreCard,
- ScoreCardsBlock,
- StatChip,
- StatGridBlock,
-)
-from .normalize import normalize_issue_for_pdf
-from .options import PdfBuildOptions
-from .adapters import SECTION_ADAPTERS
-
-
-def _build_meta(
- payload: dict[str, Any],
- opts: PdfBuildOptions,
- exported_at: str,
- all_issue_counts: dict[str, int],
- overall: Optional[int],
- included_sections: list[str],
-) -> PdfMeta:
- site = str(payload.get("site_name") or "Site Audit")
- generated_raw = str(payload.get("report_generated_at") or "")
- generated = _format_report_date(generated_raw)
- meta_block = payload.get("report_meta") or {}
- data_sources: list[str] = []
- if isinstance(meta_block, dict):
- data_sources = [str(s) for s in (meta_block.get("data_sources") or [])]
- report_title = str(payload.get("report_title") or "Technical SEO Audit Report")
- return PdfMeta(
- report_id=opts.report_id,
- property=site,
- report_title=report_title,
- generated_at=generated,
- exported_at=exported_at,
- data_sources=data_sources,
- health_score=overall,
- issue_counts=all_issue_counts,
- included_sections=included_sections,
- )
-
-
-def _build_cover(
- payload: dict[str, Any],
- opts: PdfBuildOptions,
- overall: Optional[int],
- all_issue_counts: dict[str, int],
-) -> PdfCoverBlock:
- site = str(payload.get("site_name") or "Site Audit")
- report_title = str(payload.get("report_title") or "Technical SEO Audit Report")
-
- score_txt, band = _score_band(float(overall) if overall is not None else None)
- hero = PdfScoreHero(score=score_txt, band=band, label="Overall health score") # type: ignore[arg-type]
-
- priority_chips = [
- StatChip(label="Critical", value=str(all_issue_counts["critical"]), tone="critical"),
- StatChip(label="High", value=str(all_issue_counts["high"]), tone="high"),
- StatChip(label="Medium", value=str(all_issue_counts["medium"]), tone="medium"),
- StatChip(label="Low", value=str(all_issue_counts["low"]), tone="low"),
- ]
- priority_strip = StatGridBlock(id="cover.priority_strip", chips=priority_chips, columns=4)
-
- categories = payload.get("categories") or []
- score_cards: list[ScoreCard] = []
- for cat in categories:
- if not isinstance(cat, dict):
- continue
- from ...reporting.terminology import category_display_name
- name = category_display_name(str(cat.get("name") or "Category"))
- raw = cat.get("score")
- sv: float | None = None
- if raw is not None:
- try:
- sv = float(raw)
- except (TypeError, ValueError):
- pass
- stxt, sband = _score_band(sv)
- issue_n = len(cat.get("issues") or [])
- score_cards.append(ScoreCard(name=name, score=stxt, issue_count=issue_n, tone=sband)) # type: ignore[arg-type]
- cat_scores_block = ScoreCardsBlock(id="cover.category_scores", cards=score_cards)
-
- # Executive summary
- exec_data = _executive_export_data(payload)
- exec_summary = exec_data.get("summary") or None
- exec_source = _executive_source_label(exec_data.get("source") or "") if exec_data.get("source") else None
- priorities_list: list[str] = exec_data.get("priorities") or []
-
- # Top issues for cover — one row per distinct headline; prefer rows with a URL
- all_rows = sorted(_issues_rows(payload), key=_priority_sort_key)
- top_limit = opts.limits.top_issues_cover
- headline_order: list[str] = []
- by_headline: dict[str, PdfIssue] = {}
- for row in all_rows:
- issue = normalize_issue_for_pdf(row, include_recommendation=False)
- if issue.headline not in by_headline:
- headline_order.append(issue.headline)
- by_headline[issue.headline] = issue
- elif not by_headline[issue.headline].url and issue.url:
- by_headline[issue.headline] = issue
- top_issues = [by_headline[h] for h in headline_order[:top_limit]]
-
- return PdfCoverBlock(
- headline=f"Site Audit — {site}",
- subtitle=report_title,
- hero=hero,
- priority_strip=priority_strip,
- category_scores=cat_scores_block,
- executive_summary=exec_summary,
- executive_source=exec_source,
- priorities_list=priorities_list[:8],
- top_issues=top_issues,
- )
-
-
-def build_pdf_document(
- payload: dict[str, Any],
- opts: Optional[PdfBuildOptions] = None,
-) -> PdfDocument:
- """Transform a raw ReportPayload dict into a PdfDocument ready for rendering."""
- if opts is None:
- opts = PdfBuildOptions()
-
- exported_at = datetime.now(timezone.utc).strftime("%d %B %Y, %H:%M UTC")
- overall = _overall_score(payload)
- all_issues = _issues_rows(payload)
- all_issue_counts = _issue_priority_counts(all_issues)
-
- effective_sections = opts.effective_sections()
-
- # Run each requested adapter
- sections: list = []
- for key in effective_sections:
- adapter = SECTION_ADAPTERS.get(key)
- if adapter is None:
- continue
- result = adapter(payload, opts)
- sections.extend(result)
-
- # Sort sections by priority
- sections.sort(key=lambda s: s.priority)
-
- meta = _build_meta(
- payload, opts, exported_at, all_issue_counts, overall,
- included_sections=effective_sections,
- )
- cover = _build_cover(payload, opts, overall, all_issue_counts)
- footer = PdfFooterBlock(exported_at=exported_at)
-
- return PdfDocument(
- schema_version=SCHEMA_VERSION,
- document_kind="audit",
- meta=meta,
- cover=cover,
- sections=sections,
- footer=footer,
- appendix=None, # appendix content is included as PdfSections in sections list
- )
diff --git a/src/website_profiling/reporting/pdf/document.py b/src/website_profiling/reporting/pdf/document.py
deleted file mode 100644
index 94e9aa4c..00000000
--- a/src/website_profiling/reporting/pdf/document.py
+++ /dev/null
@@ -1,322 +0,0 @@
-"""PdfDocument v1 — versioned, block-based document model.
-
-All types are JSON-serializable dataclasses. The renderer consumes these;
-no ReportLab types appear here.
-"""
-from __future__ import annotations
-
-from dataclasses import dataclass, field
-from typing import Any, Literal, Optional
-
-SCHEMA_VERSION = "1.0"
-
-# ---------------------------------------------------------------------------
-# Primitive / shared
-# ---------------------------------------------------------------------------
-
-PriorityTone = Literal["critical", "high", "medium", "low", "neutral", "good", "fair", "poor"]
-DocumentKind = Literal["audit", "compare"]
-
-
-@dataclass
-class PdfTruncation:
- shown: int
- total: int
- reason: Literal["limit", "page_budget", "empty"] = "limit"
- continue_in: list[str] = field(default_factory=lambda: ["CSV", "workbook"])
-
-
-# ---------------------------------------------------------------------------
-# Block types — renderer handles each `type` discriminator
-# ---------------------------------------------------------------------------
-
-@dataclass
-class HeadingBlock:
- type: str = field(default="heading", init=False)
- id: str = ""
- text: str = ""
- level: int = 2 # 2 = section heading, 3 = sub-heading
- visible: bool = True
-
-
-@dataclass
-class ParagraphBlock:
- type: str = field(default="paragraph", init=False)
- id: str = ""
- text: str = ""
- italic: bool = False
- visible: bool = True
-
-
-@dataclass
-class CalloutBlock:
- type: str = field(default="callout", init=False)
- id: str = ""
- text: str = ""
- severity: Literal["info", "warn", "critical"] = "info"
- visible: bool = True
-
-
-@dataclass
-class SpacerBlock:
- type: str = field(default="spacer", init=False)
- id: str = ""
- height_pt: float = 8.0
- visible: bool = True
-
-
-@dataclass
-class KpiItem:
- label: str
- value: str
- delta: Optional[str] = None
- tone: PriorityTone = "neutral"
- help: Optional[str] = None
-
-
-@dataclass
-class KpiRowBlock:
- type: str = field(default="kpi_row", init=False)
- id: str = ""
- items: list[KpiItem] = field(default_factory=list)
- visible: bool = True
-
-
-@dataclass
-class StatChip:
- label: str
- value: str
- tone: PriorityTone = "neutral"
-
-
-@dataclass
-class StatGridBlock:
- type: str = field(default="stat_grid", init=False)
- id: str = ""
- chips: list[StatChip] = field(default_factory=list)
- columns: int = 4
- visible: bool = True
-
-
-@dataclass
-class KeyValueBlock:
- type: str = field(default="key_value", init=False)
- id: str = ""
- rows: list[tuple[str, str]] = field(default_factory=list)
- layout: Literal["default", "audit", "glossary"] = "default"
- visible: bool = True
-
-
-@dataclass
-class ScoreCard:
- name: str
- score: Optional[str] # formatted string, e.g. "87" or "—"
- issue_count: int = 0
- tone: Literal["score-good", "score-fair", "score-poor", "score-na"] = "score-na"
-
-
-@dataclass
-class ScoreCardsBlock:
- type: str = field(default="score_cards", init=False)
- id: str = ""
- cards: list[ScoreCard] = field(default_factory=list)
- visible: bool = True
-
-
-@dataclass
-class TableColumn:
- key: str
- label: str
- width: Literal["narrow", "medium", "wide", "url"] = "medium"
- align: Literal["left", "center", "right"] = "left"
-
-
-@dataclass
-class MetricTableBlock:
- type: str = field(default="metric_table", init=False)
- id: str = ""
- columns: list[TableColumn] = field(default_factory=list)
- rows: list[dict[str, str]] = field(default_factory=list)
- repeat_header: bool = True
- truncation: Optional[PdfTruncation] = None
- visible: bool = True
-
-
-@dataclass
-class UrlListBlock:
- type: str = field(default="url_list", init=False)
- id: str = ""
- rows: list[dict[str, str]] = field(default_factory=list) # keys: url, status, title
- show_title: bool = True
- truncation: Optional[PdfTruncation] = None
- visible: bool = True
-
-
-# ---------------------------------------------------------------------------
-# Issue blocks — primary findings format
-# ---------------------------------------------------------------------------
-
-@dataclass
-class PdfIssueMetrics:
- gsc_clicks: Optional[int] = None
- gsc_impressions: Optional[int] = None
- ga4_sessions: Optional[int] = None
- impact_score: Optional[float] = None
- lh_audit_id: Optional[str] = None
-
-
-@dataclass
-class PdfIssue:
- id: str
- priority: str
- category: str
- headline: str # ≤ 80 chars, no embedded URL duplication
- url: Optional[str] = None
- path: Optional[str] = None # display-only short path
- detail: Optional[str] = None
- recommendation: Optional[str] = None
- metrics: Optional[PdfIssueMetrics] = None
- tags: list[str] = field(default_factory=list)
- related_urls: list[str] = field(default_factory=list) # collapsed duplicates
-
-
-@dataclass
-class IssueGroupBlock:
- type: str = field(default="issue_group", init=False)
- id: str = ""
- title: str = ""
- group_label: str = "" # e.g. "Critical — 1 issue"
- issues: list[PdfIssue] = field(default_factory=list)
- render_as: Literal["list", "compact_table"] = "list"
- truncation: Optional[PdfTruncation] = None
- visible: bool = True
-
-
-@dataclass
-class IssueTableBlock:
- """Fallback tabular rendering for dense medium/low groups."""
- type: str = field(default="issue_table", init=False)
- id: str = ""
- title: str = ""
- issues: list[PdfIssue] = field(default_factory=list)
- truncation: Optional[PdfTruncation] = None
- visible: bool = True
-
-
-@dataclass
-class MarkdownBlock:
- type: str = field(default="markdown", init=False)
- id: str = ""
- text: str = ""
- visible: bool = True
-
-
-# Union type for IDE / type-checkers
-PdfBlock = (
- HeadingBlock
- | ParagraphBlock
- | CalloutBlock
- | SpacerBlock
- | KpiRowBlock
- | StatGridBlock
- | KeyValueBlock
- | ScoreCardsBlock
- | MetricTableBlock
- | UrlListBlock
- | IssueGroupBlock
- | IssueTableBlock
- | MarkdownBlock
-)
-
-# ---------------------------------------------------------------------------
-# Cover
-# ---------------------------------------------------------------------------
-
-@dataclass
-class PdfScoreHero:
- score: Optional[str]
- band: Literal["score-good", "score-fair", "score-poor", "score-na"]
- label: str # e.g. "Overall health score"
-
-
-@dataclass
-class PdfCoverBlock:
- headline: str
- subtitle: str
- hero: PdfScoreHero
- priority_strip: StatGridBlock
- category_scores: ScoreCardsBlock
- executive_summary: Optional[str] = None # prose paragraph
- executive_source: Optional[str] = None
- priorities_list: list[str] = field(default_factory=list)
- top_issues: list[PdfIssue] = field(default_factory=list)
-
-
-# ---------------------------------------------------------------------------
-# Section
-# ---------------------------------------------------------------------------
-
-@dataclass
-class PdfSection:
- id: str
- section_key: str
- title: str
- priority: int = 50 # lower = earlier in document
- page_break_before: bool = False
- keep_with_next_blocks: int = 1
- source_label: Optional[str] = None
- provenance: Optional[str] = None
- blocks: list[Any] = field(default_factory=list) # list[PdfBlock]
- truncation: Optional[PdfTruncation] = None
-
-
-# ---------------------------------------------------------------------------
-# Appendix
-# ---------------------------------------------------------------------------
-
-@dataclass
-class PdfAppendix:
- url_sample: Optional[UrlListBlock] = None
- audit_details: Optional[KeyValueBlock] = None
- glossary: Optional[KeyValueBlock] = None
-
-
-# ---------------------------------------------------------------------------
-# Meta / Footer
-# ---------------------------------------------------------------------------
-
-@dataclass
-class PdfMeta:
- report_id: Optional[int]
- property: str
- report_title: str
- generated_at: str # formatted for display
- exported_at: str
- data_sources: list[str]
- health_score: Optional[int]
- issue_counts: dict[str, int] # {critical, high, medium, low}
- truncation_summary: list[str] = field(default_factory=list)
- included_sections: list[str] = field(default_factory=list)
- locale: str = "en"
-
-
-@dataclass
-class PdfFooterBlock:
- confidential_note: str = "Confidential — prepared for client review."
- generator: str = "Site Audit"
- exported_at: str = ""
-
-
-# ---------------------------------------------------------------------------
-# Root document
-# ---------------------------------------------------------------------------
-
-@dataclass
-class PdfDocument:
- schema_version: str
- document_kind: DocumentKind
- meta: PdfMeta
- cover: PdfCoverBlock
- sections: list[PdfSection]
- footer: PdfFooterBlock
- appendix: Optional[PdfAppendix] = None
diff --git a/src/website_profiling/reporting/pdf/normalize.py b/src/website_profiling/reporting/pdf/normalize.py
deleted file mode 100644
index b2d7547c..00000000
--- a/src/website_profiling/reporting/pdf/normalize.py
+++ /dev/null
@@ -1,358 +0,0 @@
-"""Issue normalization and grouping for PDF output.
-
-Transforms raw ``_issues_rows`` dicts (which mirror the DB payload) into
-``PdfIssue`` objects suited for print layout:
-- Strips duplicated URLs from headlines
-- Expands Lighthouse audit-id abbreviations into human labels
-- Groups by priority → category for use by IssueGroupBlock
-"""
-from __future__ import annotations
-
-import hashlib
-import re
-from typing import Any, Optional
-from urllib.parse import urlparse
-
-from .document import IssueGroupBlock, PdfIssue, PdfIssueMetrics, PdfTruncation
-
-# ---------------------------------------------------------------------------
-# Lighthouse audit-id → human label registry
-# ---------------------------------------------------------------------------
-
-_LH_AUDIT_LABELS: dict[str, str] = {
- "cache-insight": "Serve assets with efficient cache policy",
- "color-contrast": "Background and foreground colors lack sufficient contrast",
- "unused-css-rules": "Remove unused CSS",
- "errors-in-console": "Browser errors logged to the console",
- "label-content-name-mismatch": "Button/link label does not match accessible name",
- "network-dependency-tree-insight": "Minimize critical request chain depth",
- "render-blocking-insight": "Eliminate render-blocking resources",
- "unused-javascript": "Remove unused JavaScript",
- "uses-optimized-images": "Efficiently encode images",
- "uses-responsive-images": "Properly size images",
- "uses-webp-images": "Serve images in next-gen formats",
- "largest-contentful-paint-element": "Largest Contentful Paint element",
- "total-blocking-time": "Total Blocking Time",
- "cumulative-layout-shift": "Cumulative Layout Shift",
- "first-contentful-paint": "First Contentful Paint",
- "speed-index": "Speed Index",
- "interactive": "Time to Interactive",
- "server-response-time": "Reduce initial server response time",
- "dom-size": "Avoid an excessive DOM size",
- "long-tasks": "Avoid long main-thread tasks",
- "layout-shifts": "Avoid large layout shifts",
- "image-alt": "Image elements do not have alt attributes",
- "link-name": "Links do not have a discernible name",
- "button-name": "Buttons do not have an accessible name",
- "duplicate-id-active": "Document has active focus elements with duplicate ID",
- "heading-order": "Heading elements are not in a sequentially-descending order",
- "meta-description": "Document does not have a meta description",
- "document-title": "Document does not have a element",
- "hreflang": "Document does not have a valid hreflang",
- "canonical": "Page is not canonical",
- "robots-txt": "Robots.txt is not valid",
- "tap-targets": "Touch targets are not sized appropriately",
-}
-
-_URL_IN_MSG_PATTERN = re.compile(
- r"(https?://\S+|(?:^|[\s:])(/\S+))", re.IGNORECASE
-)
-
-# Colon at end of a known-bad audit id: "cache-insight:" → strip colon
-_AUDIT_ID_TRAILING_COLON = re.compile(r"^([\w-]+):$")
-
-
-def _lh_label(audit_id: str) -> str:
- """Return a human-readable label for a Lighthouse audit id."""
- clean = audit_id.rstrip(":").strip().lower()
- return _LH_AUDIT_LABELS.get(clean, clean.replace("-", " ").title())
-
-
-def _strip_url_from_headline(message: str, url: str) -> str:
- """Remove URL from message text when it duplicates the dedicated url field."""
- if not url or not message:
- return message
-
- # Direct inclusion: "Issue text: https://example.com/path"
- stripped = message.replace(url, "").strip().rstrip(":").strip()
- if stripped and stripped != message:
- return stripped
-
- # URL with trailing slash variant
- url_slash = url.rstrip("/") + "/"
- stripped2 = message.replace(url_slash, "").strip().rstrip(":").strip()
- if stripped2 and stripped2 != message:
- return stripped2
-
- return message
-
-
-def _extract_path(url: str) -> Optional[str]:
- """Return just the path component of a URL for compact display."""
- if not url:
- return None
- try:
- parsed = urlparse(url)
- return parsed.path or None
- except Exception:
- return None
-
-
-def _is_lighthouse_row(message: str, tags: list[str]) -> tuple[bool, str]:
- """Detect Lighthouse issue rows and return (is_lh, audit_id)."""
- # Pattern: "audit-id:" alone or at start of message
- m = _AUDIT_ID_TRAILING_COLON.match(message.strip())
- if m:
- return True, m.group(1)
- # Tag-based
- if "lighthouse" in tags:
- return True, ""
- return False, ""
-
-
-def _issue_id(row: dict[str, Any]) -> str:
- key = f"{row.get('category','')}\x00{row.get('priority','')}\x00{row.get('message','')}\x00{row.get('url','')}"
- return hashlib.md5(key.encode()).hexdigest()[:12]
-
-
-def _shorten_headline(headline: str, raw_message: str, url: str) -> str:
- """Apply common headline cleanups after URL strip / lighthouse expansion."""
- lower = headline.lower()
- lower_raw = raw_message.lower()
-
- if "url in sitemap but not crawled" in lower:
- return "In sitemap, not crawled"
-
- if lower_raw.startswith("redirect:"):
- m = re.match(r"redirect:\s*(\d{3})\s*to\b", lower_raw)
- if m:
- return f"{m.group(1)} redirect"
-
- if lower.startswith("lighthouse:"):
- return headline.split(":", 1)[-1].strip()
-
- if lower.startswith("axe:"):
- body = headline.split(":", 1)[-1].strip()
- if len(body) > 90:
- dot = body.find(". ")
- if dot > 0:
- body = body[: dot + 1]
- else:
- body = body[:87].rsplit(" ", 1)[0] + "…"
- return body
-
- if len(headline) > 100:
- return headline[:97].rsplit(" ", 1)[0] + "…"
-
- return headline
-
-
-_GENERIC_CWV_REC = "See Performance (Core Web Vitals) in this audit, or re-run Lighthouse from Run audit."
-
-
-def _normalize_recommendation(rec: Optional[str]) -> Optional[str]:
- if not rec:
- return None
- if rec.strip() == _GENERIC_CWV_REC:
- return "Review Lighthouse audit details for this page."
- return rec.strip()
-
-
-def collapse_duplicate_issues(issues: list[PdfIssue]) -> list[PdfIssue]:
- """Merge rows that share the same headline + recommendation into one card with URL list."""
- buckets: dict[tuple[str, str], list[PdfIssue]] = {}
- order: list[tuple[str, str]] = []
- for iss in issues:
- key = (iss.headline, iss.recommendation or "")
- if key not in buckets:
- order.append(key)
- buckets[key] = []
- buckets[key].append(iss)
-
- collapsed: list[PdfIssue] = []
- for key in order:
- group = buckets[key]
- first = group[0]
- urls: list[str] = []
- for item in group:
- if item.url and item.url not in urls:
- urls.append(item.url)
- if len(urls) <= 1:
- collapsed.append(first)
- continue
- headline = first.headline
- if len(urls) > 1 and not headline.endswith(")"):
- headline = f"{headline} ({len(urls)} URLs)"
- collapsed.append(PdfIssue(
- id=first.id,
- priority=first.priority,
- category=first.category,
- headline=headline,
- url=None,
- path=first.path,
- detail=first.detail,
- recommendation=first.recommendation,
- metrics=first.metrics,
- tags=first.tags,
- related_urls=urls,
- ))
- return collapsed
-
-
-def normalize_issue_for_pdf(
- row: dict[str, Any],
- include_recommendation: bool = True,
-) -> PdfIssue:
- """Convert a raw issues_row dict → PdfIssue for print layout."""
- priority = str(row.get("priority") or "").lower()
- category = str(row.get("category") or "")
- raw_message = str(row.get("message") or "").strip()
- url = str(row.get("url") or "").strip()
- recommendation = _normalize_recommendation(
- str(row.get("recommendation") or "").strip() if include_recommendation else None
- )
-
- # Detect Lighthouse rows (audit-id only, no human label). Pass the row's own
- # tags so tag-based detection actually works (was hardcoded to [], making the
- # `"lighthouse" in tags` branch dead).
- is_lh, audit_id = _is_lighthouse_row(
- raw_message, [str(t).lower() for t in (row.get("tags") or [])]
- )
- if is_lh and audit_id:
- headline = _lh_label(audit_id)
- else:
- headline = _strip_url_from_headline(raw_message, url)
-
- headline = _shorten_headline(headline, raw_message, url)
-
- tags: list[str] = []
- lower_msg = raw_message.lower()
- if "sitemap" in lower_msg:
- tags.append("sitemap")
- if is_lh or "lighthouse" in lower_msg:
- tags.append("lighthouse")
- if "axe" in lower_msg or "wcag" in lower_msg or "contrast" in lower_msg:
- tags.append("axe")
- if "redirect" in lower_msg:
- tags.append("redirect")
- if "canonical" in lower_msg:
- tags.append("canonical")
- if "security" in category.lower():
- tags.append("security")
-
- # Metrics from issue dict (ReportIssue fields)
- gsc_clicks = row.get("gsc_clicks")
- gsc_imp = row.get("gsc_impressions")
- impact = row.get("impact_score")
- lh_id = audit_id if is_lh else row.get("lh_audit_id")
- metrics = None
- if any(v is not None for v in (gsc_clicks, gsc_imp, impact, lh_id)):
- metrics = PdfIssueMetrics(
- gsc_clicks=int(gsc_clicks) if gsc_clicks is not None else None,
- gsc_impressions=int(gsc_imp) if gsc_imp is not None else None,
- impact_score=float(impact) if impact is not None else None,
- lh_audit_id=str(lh_id) if lh_id else None,
- )
-
- return PdfIssue(
- id=_issue_id(row),
- priority=priority,
- category=category,
- headline=headline,
- url=url or None,
- path=_extract_path(url),
- detail=None,
- recommendation=recommendation or None,
- metrics=metrics,
- tags=tags,
- )
-
-
-# ---------------------------------------------------------------------------
-# Grouping
-# ---------------------------------------------------------------------------
-
-_PRIORITY_ORDER = {"critical": 0, "high": 1, "medium": 2, "low": 3}
-_PRIORITY_LABELS = {
- "critical": "Critical",
- "high": "High",
- "medium": "Medium",
- "low": "Low",
-}
-
-# Above this count per priority, sub-group by category
-_SUBGROUP_THRESHOLD = 8
-
-# Always use stacked list layout — tables only for cover top-issues / URL inventory
-_COMPACT_TABLE_THRESHOLD = 999
-
-
-def group_issues_for_pdf(
- issues: list[PdfIssue],
- issues_per_group: int = 25,
- issues_total: int = 120,
-) -> list[IssueGroupBlock]:
- """Group PdfIssue list by priority → category, returning IssueGroupBlock list."""
- # Sort and cap total
- sorted_issues = sorted(issues, key=lambda i: (_PRIORITY_ORDER.get(i.priority, 9), i.category))
- if len(sorted_issues) > issues_total:
- sorted_issues = sorted_issues[:issues_total]
-
- # Bucket by priority
- by_priority: dict[str, list[PdfIssue]] = {}
- for iss in sorted_issues:
- by_priority.setdefault(iss.priority, []).append(iss)
-
- groups: list[IssueGroupBlock] = []
-
- for pri in ("critical", "high", "medium", "low"):
- pri_issues = by_priority.get(pri, [])
- if not pri_issues:
- continue
-
- pri_label = _PRIORITY_LABELS.get(pri, pri.title())
- total_in_pri = len(pri_issues)
-
- if total_in_pri <= _SUBGROUP_THRESHOLD:
- # Single group for this priority
- shown = collapse_duplicate_issues(pri_issues[:issues_per_group])
- trunc = (
- PdfTruncation(shown=len(shown), total=total_in_pri)
- if total_in_pri > len(shown)
- else None
- )
- render_as = "compact_table" if len(shown) >= _COMPACT_TABLE_THRESHOLD else "list"
- groups.append(IssueGroupBlock(
- id=f"findings.{pri}",
- title=f"{pri_label} findings",
- group_label=f"{pri_label} — {total_in_pri} issue{'s' if total_in_pri != 1 else ''}",
- issues=shown,
- render_as=render_as,
- truncation=trunc,
- ))
- else:
- # Sub-group by category
- by_cat: dict[str, list[PdfIssue]] = {}
- for iss in pri_issues:
- by_cat.setdefault(iss.category, []).append(iss)
-
- for cat, cat_issues in sorted(by_cat.items()):
- cat_total = len(cat_issues)
- shown = collapse_duplicate_issues(cat_issues[:issues_per_group])
- trunc = (
- PdfTruncation(shown=len(shown), total=cat_total)
- if cat_total > len(shown)
- else None
- )
- render_as = "compact_table" if len(shown) >= _COMPACT_TABLE_THRESHOLD else "list"
- cat_id = cat.lower().replace(" ", "_").replace("&", "and")
- groups.append(IssueGroupBlock(
- id=f"findings.{pri}.{cat_id}",
- title=f"{pri_label} — {cat}",
- group_label=f"{pri_label} — {cat}: {cat_total} issue{'s' if cat_total != 1 else ''}",
- issues=shown,
- render_as=render_as,
- truncation=trunc,
- ))
-
- return groups
diff --git a/src/website_profiling/reporting/pdf/options.py b/src/website_profiling/reporting/pdf/options.py
deleted file mode 100644
index 678a446a..00000000
--- a/src/website_profiling/reporting/pdf/options.py
+++ /dev/null
@@ -1,43 +0,0 @@
-"""PdfBuildOptions, PdfLimits, and document profiles."""
-from __future__ import annotations
-
-from dataclasses import dataclass, field
-from typing import Literal, Optional
-
-Profile = Literal["executive", "standard", "full"]
-
-# Sections for each profile; None in sections means "use profile default"
-_PROFILE_SECTIONS: dict[str, list[str]] = {
- "executive": ["core"],
- "standard": ["core", "findings", "appendix"],
- "full": ["core", "findings", "lighthouse", "security", "traffic", "keywords",
- "indexation", "content", "links", "appendix"],
-}
-
-
-@dataclass
-class PdfLimits:
- issues_total: int = 120
- issues_per_group: int = 25
- top_issues_cover: int = 6
- urls_sample: int = 20
- metric_table_rows: int = 15
- gsc_queries: int = 10
- keyword_rows: int = 15
- diagnostic_items: int = 20
-
-
-@dataclass
-class PdfBuildOptions:
- profile: Profile = "standard"
- sections: Optional[list[str]] = None # None → derive from profile
- limits: PdfLimits = field(default_factory=PdfLimits)
- include_appendix: bool = True
- include_recommendations: bool = True
- include_glossary: bool = True
- report_id: Optional[int] = None
-
- def effective_sections(self) -> list[str]:
- if self.sections is not None:
- return self.sections
- return _PROFILE_SECTIONS.get(self.profile, _PROFILE_SECTIONS["standard"])
diff --git a/src/website_profiling/reporting/pdf/render/__init__.py b/src/website_profiling/reporting/pdf/render/__init__.py
deleted file mode 100644
index fc2a79ad..00000000
--- a/src/website_profiling/reporting/pdf/render/__init__.py
+++ /dev/null
@@ -1,7 +0,0 @@
-"""PDF/HTML renderers."""
-from __future__ import annotations
-
-from .html import render_html_document
-from .reportlab import render_pdf_document
-
-__all__ = ["render_pdf_document", "render_html_document"]
diff --git a/src/website_profiling/reporting/pdf/render/html.py b/src/website_profiling/reporting/pdf/render/html.py
deleted file mode 100644
index ee1eda8f..00000000
--- a/src/website_profiling/reporting/pdf/render/html.py
+++ /dev/null
@@ -1,774 +0,0 @@
-"""HTML renderer — converts PdfDocument → preview/print HTML matching the PDF layout."""
-from __future__ import annotations
-
-import html
-import re
-from typing import Any
-
-from ..document import (
- CalloutBlock,
- HeadingBlock,
- IssueGroupBlock,
- IssueTableBlock,
- KeyValueBlock,
- KpiRowBlock,
- MarkdownBlock,
- MetricTableBlock,
- ParagraphBlock,
- PdfCoverBlock,
- PdfDocument,
- PdfIssue,
- PdfSection,
- ScoreCardsBlock,
- SpacerBlock,
- StatGridBlock,
- UrlListBlock,
-)
-from . import styles as S
-
-
-def html_styles() -> str:
- """CSS shared by standard export preview HTML."""
- return """
- :root {
- --ink: #0f172a;
- --muted: #64748b;
- --line: #e2e8f0;
- --surface: #ffffff;
- --surface-muted: #f8fafc;
- --header-bg: #f1f5f9;
- --brand-accent: #2563eb;
- --good: #059669;
- --good-bg: #ecfdf5;
- --fair: #d97706;
- --fair-bg: #fffbeb;
- --poor: #dc2626;
- --poor-bg: #fef2f2;
- --critical-fg: #991b1b;
- --critical-bg: #fee2e2;
- --high-fg: #c2410c;
- --high-bg: #ffedd5;
- --medium-fg: #a16207;
- --medium-bg: #fef3c7;
- --low-fg: #475569;
- --low-bg: #f1f5f9;
- }
- * { box-sizing: border-box; }
- body {
- margin: 0;
- background: #eef2f7;
- color: var(--ink);
- font: 400 14px/1.45 "Segoe UI", system-ui, -apple-system, sans-serif;
- }
- .report {
- max-width: 816px;
- margin: 0 auto;
- background: var(--surface);
- box-shadow: 0 1px 3px rgba(15, 23, 42, 0.08);
- }
- .cover {
- padding: 1.75rem 1.85rem 1.25rem;
- background: var(--surface);
- }
- .cover-head {
- display: flex;
- justify-content: space-between;
- align-items: flex-start;
- gap: 1rem;
- margin-bottom: 0.35rem;
- }
- .cover-head h1 {
- margin: 0;
- font-size: 1.35rem;
- font-weight: 700;
- line-height: 1.25;
- }
- .cover-subtitle {
- margin: 0.25rem 0 0;
- color: var(--muted);
- font-size: 0.92rem;
- }
- .hero-score {
- text-align: center;
- min-width: 4.5rem;
- }
- .hero-score .score {
- display: block;
- font-size: 2rem;
- font-weight: 700;
- line-height: 1;
- }
- .hero-score .suffix {
- display: block;
- margin-top: 0.15rem;
- font-size: 0.72rem;
- color: var(--muted);
- }
- .hero-score.score-good .score { color: var(--good); }
- .hero-score.score-fair .score { color: var(--fair); }
- .hero-score.score-poor .score { color: var(--poor); }
- .hero-score.score-na .score { color: var(--muted); }
- .cover-meta-line {
- margin: 0.5rem 0 1rem;
- color: var(--muted);
- font-size: 0.82rem;
- }
- .section-title {
- margin: 1.1rem 0 0.35rem;
- font-size: 0.82rem;
- font-weight: 700;
- color: var(--ink);
- }
- .section-rule {
- border: none;
- border-top: 1px solid var(--line);
- margin: 0 0 0.65rem;
- }
- .section-lead {
- margin: 0 0 0.65rem;
- color: var(--muted);
- font-size: 0.78rem;
- }
- .grid-table {
- width: 100%;
- border-collapse: collapse;
- table-layout: fixed;
- margin-bottom: 0.85rem;
- font-size: 0.82rem;
- }
- .grid-table th,
- .grid-table td {
- border: 1px solid var(--line);
- padding: 0.65rem 0.5rem;
- text-align: center;
- vertical-align: middle;
- }
- .stat-grid td.stat-critical { background: var(--critical-bg); color: var(--critical-fg); }
- .stat-grid td.stat-high { background: var(--high-bg); color: var(--high-fg); }
- .stat-grid td.stat-medium { background: var(--medium-bg); color: var(--medium-fg); }
- .stat-grid td.stat-low { background: var(--low-bg); color: var(--low-fg); }
- .stat-grid .stat-value {
- display: block;
- font-size: 1.15rem;
- font-weight: 700;
- line-height: 1.1;
- }
- .stat-grid .stat-label {
- display: block;
- margin-top: 0.2rem;
- font-size: 0.72rem;
- color: var(--muted);
- }
- .score-grid .score-value {
- display: block;
- font-size: 0.95rem;
- font-weight: 700;
- line-height: 1.1;
- }
- .score-grid .score-name {
- display: block;
- margin-top: 0.25rem;
- font-size: 0.72rem;
- font-weight: 600;
- line-height: 1.25;
- }
- .score-grid .score-meta {
- display: block;
- margin-top: 0.15rem;
- font-size: 0.68rem;
- color: var(--muted);
- }
- .score-grid td { background: var(--surface-muted); }
- .score-grid .score-good .score-value { color: var(--good); }
- .score-grid .score-fair .score-value { color: var(--fair); }
- .score-grid .score-poor .score-value { color: var(--poor); }
- .score-grid .score-na .score-value { color: var(--muted); }
- .exec-panel {
- border: 1px solid var(--line);
- border-left: 3px solid var(--brand-accent);
- background: var(--surface-muted);
- padding: 0.85rem 1rem;
- margin-bottom: 1rem;
- border-radius: 0 4px 4px 0;
- }
- .exec-source {
- margin: 0 0 0.45rem;
- font-size: 0.68rem;
- font-weight: 700;
- color: var(--brand-accent);
- text-transform: uppercase;
- letter-spacing: 0.04em;
- }
- .exec-body { margin: 0; font-size: 0.88rem; line-height: 1.5; }
- .exec-subhead {
- margin: 0.65rem 0 0.35rem;
- font-size: 0.72rem;
- font-weight: 700;
- color: var(--muted);
- }
- .exec-priorities {
- margin: 0;
- padding-left: 1.1rem;
- font-size: 0.82rem;
- line-height: 1.45;
- }
- .data-table {
- width: 100%;
- border-collapse: collapse;
- font-size: 0.82rem;
- border: 1px solid var(--line);
- margin-bottom: 0.85rem;
- }
- .data-table th,
- .data-table td {
- padding: 0.55rem 0.65rem;
- text-align: left;
- vertical-align: middle;
- border-bottom: 1px solid var(--line);
- }
- .data-table thead th {
- background: var(--header-bg);
- font-size: 0.72rem;
- font-weight: 700;
- color: var(--muted);
- }
- .data-table tbody tr:nth-child(even) td { background: var(--surface-muted); }
- .data-table tbody tr:last-child td { border-bottom: none; }
- .data-table .col-status { text-align: center; width: 4.5rem; }
- .data-table .col-priority { text-align: center; width: 5rem; }
- .kv-audit th {
- width: 23%;
- font-weight: 700;
- vertical-align: top;
- }
- .kv-glossary th {
- width: 21%;
- font-weight: 700;
- vertical-align: top;
- background: var(--header-bg);
- }
- .kv-glossary td { line-height: 1.45; }
- .link { color: var(--brand-accent); word-break: break-all; }
- .site-wide { color: var(--muted); font-style: italic; font-size: 0.78rem; }
- .badge {
- display: inline-block;
- padding: 0.15rem 0.45rem;
- border-radius: 3px;
- font-size: 0.65rem;
- font-weight: 700;
- text-transform: uppercase;
- letter-spacing: 0.03em;
- border: 1px solid transparent;
- }
- .badge-critical { background: var(--critical-bg); color: var(--critical-fg); border-color: var(--critical-fg); }
- .badge-high { background: var(--high-bg); color: var(--high-fg); border-color: var(--high-fg); }
- .badge-medium { background: var(--medium-bg); color: var(--medium-fg); border-color: var(--medium-fg); }
- .badge-low { background: var(--low-bg); color: var(--low-fg); border-color: var(--low-fg); }
- .status-200 { background: var(--good-bg); color: var(--good); border-color: var(--good); }
- .status-3xx { background: var(--fair-bg); color: var(--fair); border-color: var(--fair); }
- .status-4xx, .status-5xx { background: var(--poor-bg); color: var(--poor); border-color: var(--poor); }
- .status-other { background: var(--surface-muted); color: var(--muted); border-color: var(--line); }
- .content { padding: 0 1.85rem 1.5rem; }
- .doc-section { margin-bottom: 1.35rem; }
- .doc-section > h2 {
- margin: 0 0 0.35rem;
- font-size: 0.82rem;
- font-weight: 700;
- }
- .doc-section .source-label {
- margin: 0 0 0.5rem;
- font-size: 0.78rem;
- color: var(--muted);
- }
- .group-label {
- margin: 0.65rem 0 0.35rem;
- font-size: 0.78rem;
- font-weight: 700;
- }
- .issue-card {
- border-left: 3px solid var(--line);
- background: var(--surface-muted);
- padding: 0.45rem 0.65rem;
- margin-bottom: 0.45rem;
- font-size: 0.82rem;
- }
- .issue-card.priority-critical { border-color: var(--critical-fg); background: var(--critical-bg); }
- .issue-card.priority-high { border-color: var(--high-fg); background: var(--high-bg); }
- .issue-card.priority-medium { border-color: var(--medium-fg); background: var(--medium-bg); }
- .issue-card.priority-low { border-color: var(--low-fg); background: var(--low-bg); }
- .issue-headline { margin: 0; font-weight: 700; line-height: 1.35; }
- .issue-url {
- margin: 0.2rem 0 0;
- font-size: 0.76rem;
- color: var(--brand-accent);
- word-break: break-all;
- }
- .issue-rec {
- margin: 0.25rem 0 0;
- font-size: 0.76rem;
- color: var(--muted);
- font-style: italic;
- }
- .issue-url-list {
- margin: 0.25rem 0 0;
- padding-left: 1rem;
- font-size: 0.76rem;
- color: var(--brand-accent);
- }
- .muted-note {
- margin: 0.35rem 0 0;
- font-size: 0.76rem;
- color: var(--muted);
- }
- .page-break {
- break-before: page;
- page-break-before: always;
- height: 0;
- margin: 0;
- border-top: 1px dashed var(--line);
- }
- .report-footer {
- border-top: 1px solid var(--line);
- padding: 0.85rem 1.85rem 1.25rem;
- color: var(--muted);
- font-size: 0.72rem;
- line-height: 1.45;
- }
- .content {
- padding: 0 1.85rem 1.5rem;
- }
- .custom-section {
- margin-bottom: 1.35rem;
- }
- .custom-section > h2 {
- margin: 0 0 0.35rem;
- font-size: 0.82rem;
- font-weight: 700;
- }
- .callout {
- border: 1px solid var(--line);
- border-left: 3px solid var(--brand-accent);
- background: var(--surface-muted);
- padding: 0.85rem 1rem;
- border-radius: 0 4px 4px 0;
- margin: 0.5rem 0;
- }
- p.muted, .muted {
- color: var(--muted);
- font-size: 0.82rem;
- margin: 0.35rem 0 0.65rem;
- }
- .url, td.url {
- color: var(--brand-accent);
- word-break: break-all;
- font-size: 0.76rem;
- }
- table.data, .table-wrap table {
- width: 100%;
- border-collapse: collapse;
- font-size: 0.82rem;
- border: 1px solid var(--line);
- margin: 0.5rem 0 0.85rem;
- }
- table.data th, table.data td,
- .table-wrap table th, .table-wrap table td {
- padding: 0.55rem 0.65rem;
- text-align: left;
- vertical-align: top;
- border-bottom: 1px solid var(--line);
- }
- table.data thead th, .table-wrap table thead th {
- background: var(--header-bg);
- font-size: 0.72rem;
- font-weight: 700;
- color: var(--muted);
- }
- .category-cards {
- display: flex;
- flex-wrap: wrap;
- gap: 0.75rem;
- margin: 0.65rem 0;
- }
- article.score-card {
- flex: 1 1 140px;
- max-width: 180px;
- border: 1px solid var(--line);
- border-radius: 4px;
- padding: 0.75rem;
- background: var(--surface-muted);
- text-align: center;
- }
- article.score-card .score-value {
- font-size: 1.1rem;
- font-weight: 700;
- }
- article.score-card .score-name {
- margin-top: 0.35rem;
- font-size: 0.72rem;
- font-weight: 600;
- }
- article.score-card .score-meta {
- margin-top: 0.2rem;
- font-size: 0.68rem;
- color: var(--muted);
- }
- article.score-card.score-good .score-value { color: var(--good); }
- article.score-card.score-fair .score-value { color: var(--fair); }
- article.score-card.score-poor .score-value { color: var(--poor); }
- article.score-card.score-na .score-value { color: var(--muted); }
- .notes, .json-preview {
- line-height: 1.5;
- font-size: 0.82rem;
- }
- .json-preview {
- overflow-x: auto;
- background: var(--surface-muted);
- padding: 0.75rem;
- border: 1px solid var(--line);
- border-radius: 4px;
- }
- @media print {
- body { background: #fff; }
- .report { max-width: none; box-shadow: none; }
- .cover, .content, .report-footer { padding-left: 0.65in; padding-right: 0.65in; }
- .page-break { border: none; }
- }
-"""
-
-
-def _esc(text: Any) -> str:
- return html.escape(str(text) if text is not None else "")
-
-
-def _priority_badge(priority: str) -> str:
- key = priority.lower()
- cls = f"badge badge-{key}" if key in {"critical", "high", "medium", "low"} else "badge badge-low"
- return f'<span class="{cls}">{_esc(priority)}</span>'
-
-
-def _status_badge(code: str) -> str:
- c = str(code or "").strip()
- if c == "200":
- cls = "badge status-200"
- elif c.startswith("3"):
- cls = "badge status-3xx"
- elif c and c[0] in "45":
- cls = "badge status-4xx" if c.startswith("4") else "badge status-5xx"
- else:
- cls = "badge status-other"
- return f'<span class="{cls}">{_esc(c or "—")}</span>'
-
-
-def _issue_location(issue: PdfIssue) -> str:
- if issue.path:
- return f'<span class="link">{_esc(issue.path)}</span>'
- if issue.url:
- return f'<span class="link">{_esc(issue.url)}</span>'
- return '<span class="site-wide">Site-wide</span>'
-
-
-def _section_heading(title: str) -> str:
- return f'<h3 class="section-title">{_esc(title)}</h3><hr class="section-rule"/>'
-
-
-def _render_stat_grid(block: StatGridBlock) -> str:
- if not block.chips:
- return ""
- cells = []
- for chip in block.chips:
- tone = chip.tone if chip.tone in {"critical", "high", "medium", "low"} else "low"
- cells.append(
- f'<td class="stat-{tone}">'
- f'<span class="stat-value">{_esc(chip.value)}</span>'
- f'<span class="stat-label">{_esc(chip.label)}</span>'
- f"</td>"
- )
- while len(cells) < block.columns:
- cells.append("<td></td>")
- return f'<table class="grid-table stat-grid"><tr>{"".join(cells)}</tr></table>'
-
-
-def _render_score_cards(block: ScoreCardsBlock) -> str:
- if not block.cards:
- return ""
- cols = S.GRID_COLS
- rows_html: list[str] = []
- row: list[str] = []
- for card in block.cards:
- issue_label = f"{card.issue_count} issue{'s' if card.issue_count != 1 else ''}"
- row.append(
- f'<td class="{_esc(card.tone)}">'
- f'<span class="score-value">{_esc(card.score or "—")}</span>'
- f'<span class="score-name">{_esc(card.name)}</span>'
- f'<span class="score-meta">{issue_label}</span>'
- f"</td>"
- )
- if len(row) == cols:
- rows_html.append(f"<tr>{''.join(row)}</tr>")
- row = []
- if row:
- while len(row) < cols:
- row.append("<td></td>")
- rows_html.append(f"<tr>{''.join(row)}</tr>")
- return f'<table class="grid-table score-grid">{"".join(rows_html)}</table>'
-
-
-def _render_executive_panel(cover: PdfCoverBlock) -> str:
- if not (cover.executive_summary or cover.priorities_list):
- return ""
- parts = ['<div class="exec-panel">']
- if cover.executive_source:
- parts.append(f'<p class="exec-source">Source · {_esc(cover.executive_source)}</p>')
- if cover.executive_summary:
- parts.append(f'<p class="exec-body">{_esc(cover.executive_summary)}</p>')
- if cover.priorities_list:
- parts.append('<p class="exec-subhead">Recommended priorities</p>')
- parts.append('<ol class="exec-priorities">')
- for pri in cover.priorities_list[:6]:
- parts.append(f"<li>{_esc(pri)}</li>")
- parts.append("</ol>")
- parts.append("</div>")
- return "".join(parts)
-
-
-def _render_top_issues(issues: list[PdfIssue]) -> str:
- if not issues:
- return ""
- rows = "".join(
- f"<tr>"
- f'<td class="col-priority">{_priority_badge(iss.priority)}</td>'
- f"<td>{_esc(iss.headline)}</td>"
- f"<td>{_issue_location(iss)}</td>"
- f"</tr>"
- for iss in issues
- )
- return (
- f"{_section_heading('Top traffic-impacting issues')}"
- f'<p class="section-lead">Ranked by severity and traffic impact — address critical and high items first.</p>'
- f'<table class="data-table top-issues">'
- f"<thead><tr><th class=\"col-priority\">Priority</th><th>Issue</th><th>Location</th></tr></thead>"
- f"<tbody>{rows}</tbody></table>"
- )
-
-
-def _render_cover(cover: PdfCoverBlock, meta) -> str:
- counts = meta.issue_counts
- total = sum(counts.values())
- meta_line = (
- f"Report generated {meta.generated_at} · {total} findings "
- f"(Critical {counts.get('critical', 0)}, High {counts.get('high', 0)}, "
- f"Medium {counts.get('medium', 0)}, Low {counts.get('low', 0)})"
- )
- hero = cover.hero
- exec_html = ""
- if cover.executive_summary or cover.priorities_list:
- exec_html = _section_heading("Executive summary") + _render_executive_panel(cover)
- top_html = _render_top_issues(cover.top_issues)
-
- cat_html = ""
- if cover.category_scores.cards:
- cat_html = _section_heading("Category scores") + _render_score_cards(cover.category_scores)
-
- return f"""
- <header class="cover">
- <div class="cover-head">
- <div>
- <h1>{_esc(cover.headline)}</h1>
- <p class="cover-subtitle">{_esc(cover.subtitle)}</p>
- </div>
- <div class="hero-score {_esc(hero.band)}">
- <span class="score">{_esc(hero.score or "—")}</span>
- <span class="suffix">/100</span>
- </div>
- </div>
- <p class="cover-meta-line">{_esc(meta_line)}</p>
- {_render_stat_grid(cover.priority_strip)}
- {cat_html}
- {exec_html}
- {top_html}
- </header>"""
-
-
-def _render_issue(issue: PdfIssue) -> str:
- pri = issue.priority.lower()
- cls = f"issue-card priority-{pri}" if pri in {"critical", "high", "medium", "low"} else "issue-card"
- parts = [f'<div class="{cls}">', f'<p class="issue-headline">{_esc(issue.headline)}</p>']
- if issue.related_urls:
- items = "".join(f"<li>{_esc(u)}</li>" for u in issue.related_urls[:10])
- extra = len(issue.related_urls) - 10
- if extra > 0:
- items += f'<li class="muted-note">… and {extra} more (see CSV export)</li>'
- parts.append(f'<ul class="issue-url-list">{items}</ul>')
- elif issue.url:
- parts.append(f'<p class="issue-url">{_esc(issue.url)}</p>')
- if issue.recommendation:
- parts.append(f'<p class="issue-rec">Fix: {_esc(issue.recommendation)}</p>')
- parts.append("</div>")
- return "".join(parts)
-
-
-def _render_issue_group(block: IssueGroupBlock) -> str:
- parts = [f'<p class="group-label">{_esc(block.group_label)}</p>']
- if block.render_as == "compact_table":
- rows = "".join(
- f"<tr><td>{_esc(iss.headline)}</td>"
- f'<td class="link">{_esc(iss.url or "")}</td></tr>'
- for iss in block.issues
- )
- parts.append(
- f'<table class="data-table"><thead><tr><th>Issue</th><th>URL</th></tr></thead>'
- f"<tbody>{rows}</tbody></table>"
- )
- else:
- for iss in block.issues:
- parts.append(_render_issue(iss))
- if block.truncation:
- t = block.truncation
- parts.append(
- f'<p class="muted-note">Showing {t.shown} of {t.total}. '
- f"Full list in {', '.join(t.continue_in)}.</p>"
- )
- return "".join(parts)
-
-
-def _render_key_value(block: KeyValueBlock) -> str:
- if not block.rows:
- return ""
- layout = getattr(block, "layout", "default") or "default"
- if layout == "audit":
- table_cls = "data-table kv-audit"
- elif layout == "glossary":
- table_cls = "data-table kv-glossary"
- else:
- table_cls = "data-table kv-audit"
- rows = "".join(
- f"<tr><th>{_esc(k)}</th><td>{_esc(v)}</td></tr>" for k, v in block.rows
- )
- return f'<table class="{table_cls}"><tbody>{rows}</tbody></table>'
-
-
-def _render_url_list(block: UrlListBlock) -> str:
- if not block.rows:
- return ""
- show_title = getattr(block, "show_title", True)
- head = "<th>URL</th><th class=\"col-status\">Status</th>"
- if show_title:
- head += "<th>Title</th>"
- body_rows: list[str] = []
- for row in block.rows:
- url = str(row.get("url") or "")
- status = str(row.get("status") or "")
- cells = (
- f'<td class="link">{_esc(url)}</td>'
- f'<td class="col-status">{_status_badge(status)}</td>'
- )
- if show_title:
- title = str(row.get("title") or "").strip()
- title_cell = _esc(title) if title else '<span class="site-wide">—</span>'
- cells += f"<td>{title_cell}</td>"
- body_rows.append(f"<tr>{cells}</tr>")
- note = ""
- if block.truncation:
- t = block.truncation
- note = (
- f'<p class="muted-note">Showing {t.shown} of {t.total} URLs. '
- f"Export CSV/workbook for full inventory.</p>"
- )
- return (
- f'<table class="data-table url-list"><thead><tr>{head}</tr></thead>'
- f'<tbody>{"".join(body_rows)}</tbody></table>{note}'
- )
-
-
-def _render_block(block: Any) -> str:
- if not getattr(block, "visible", True):
- return ""
- btype = getattr(block, "type", None)
- if btype == "issue_group":
- return _render_issue_group(block)
- if btype == "key_value":
- return _render_key_value(block)
- if btype == "url_list":
- return _render_url_list(block)
- if btype == "issue_table":
- rows = "".join(
- f"<tr><td>{_esc(iss.headline)}</td><td class=\"link\">{_esc(iss.url or '')}</td></tr>"
- for iss in block.issues
- )
- title = f"<p class=\"group-label\">{_esc(block.title)}</p>" if block.title else ""
- return (
- f"{title}<table class=\"data-table\"><thead><tr><th>Issue</th><th>URL</th></tr></thead>"
- f"<tbody>{rows}</tbody></table>"
- )
- if btype == "paragraph":
- return f"<p>{_esc(block.text)}</p>"
- if btype == "heading":
- tag = "h3" if block.level >= 3 else "h2"
- return f"<{tag}>{_esc(block.text)}</{tag}>"
- if btype == "callout":
- return f'<div class="exec-panel"><p class="exec-body">{_esc(block.text)}</p></div>'
- if btype == "markdown":
- text = re.sub(r"<[^>]+>", " ", block.text)
- return f"<p>{_esc(text)}</p>"
- if btype == "metric_table":
- cols = block.columns
- if not cols:
- return ""
- head = "".join(f"<th>{_esc(c.label)}</th>" for c in cols)
- body = ""
- for row in block.rows:
- body += "<tr>" + "".join(
- f'<td>{_esc(row.get(c.key, ""))}</td>' for c in cols
- ) + "</tr>"
- return f'<table class="data-table"><thead><tr>{head}</tr></thead><tbody>{body}</tbody></table>'
- if btype in {"spacer", "kpi_row", "stat_grid", "score_cards"}:
- return ""
- return ""
-
-
-def _render_section(section: PdfSection) -> str:
- parts = [f'<section class="doc-section" id="{_esc(section.id)}">']
- parts.append(f"<h2>{_esc(section.title)}</h2><hr class=\"section-rule\"/>")
- if section.source_label:
- parts.append(f'<p class="source-label">Source: {_esc(section.source_label)}</p>')
- for block in section.blocks:
- parts.append(_render_block(block))
- if section.truncation:
- t = section.truncation
- parts.append(
- f'<p class="muted-note">Showing {t.shown} of {t.total} issues. '
- f"Export CSV or workbook for full data.</p>"
- )
- parts.append("</section>")
- return "".join(parts)
-
-
-def render_html_document(doc: PdfDocument) -> str:
- """Render a PdfDocument as HTML matching the PDF export layout."""
- cover_html = _render_cover(doc.cover, doc.meta)
- sections_html = "".join(_render_section(s) for s in doc.sections)
- footer = doc.footer
- footer_text = (
- f"{footer.confidential_note} "
- f"Generated by {footer.generator} · {footer.exported_at}"
- )
- title = _esc(doc.cover.headline)
- return f"""<!DOCTYPE html>
-<html lang="en">
-<head>
-<meta charset="utf-8"/>
-<meta name="viewport" content="width=device-width, initial-scale=1"/>
-<title>{title}
-
-
-
-
-{cover_html}
-
-
-{sections_html}
-
-
-
-
-