diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index ea0299491..9fa810cca 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -14,7 +14,7 @@ permissions: actions: read jobs: - analyze: + analyze-actions: name: Analyze (actions) runs-on: ubuntu-latest timeout-minutes: 30 @@ -32,3 +32,22 @@ jobs: uses: github/codeql-action/analyze@v4 with: category: /language:actions + + analyze-python: + name: Analyze (python) + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v4 + with: + languages: python + build-mode: none + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v4 + with: + category: /language:python diff --git a/.gitignore b/.gitignore index 23d6b5655..551ed6ef3 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,12 @@ __pycache__ .DS_Store .env* .venv/ +.claude/ +.codex/ logs/ +examples/pifs_workspace/ +examples/ +/CONTEXT.md +/docs/pifs-core-alignment/ +/pyproject.toml +/uv.lock diff --git a/README.md b/README.md index 594b70e8e..cec74ee24 100644 --- a/README.md +++ b/README.md @@ -175,7 +175,7 @@ python3 run_pageindex.py --pdf_path /path/to/your/document.pdf You can customize the processing with additional optional arguments: ``` ---model LLM model to use (default: gpt-4o-2024-11-20) +--model LLM model to use (default: gpt-5.4) --toc-check-pages Pages to check for table of contents (default: 20) --max-pages-per-node Max pages per node (default: 10) --max-tokens-per-node Max tokens per node (default: 20000) diff --git a/examples/pifs_demo.py b/examples/pifs_demo.py new file mode 100644 index 000000000..9a7e22a78 --- /dev/null +++ b/examples/pifs_demo.py @@ -0,0 +1,46 @@ +"""A minimal PageIndex FileSystem CLI walkthrough. + +Configure the Summary Embedding Profile in your PIFS config and provide its API +key through ``PIFS_EMBEDDING_API_KEY`` before running this example. +""" + +from __future__ import annotations + +import argparse +import subprocess +from pathlib import Path + + +def run(workspace: Path, document: Path, query: str) -> int: + target = f"/documents/{document.name}" + commands = [ + ["add", str(document), "/documents"], + ["tree", "/documents", "-L", "1"], + ["browse", "/documents", query], + ["stat", target], + ["cat", target, "--structure"], + ["cat", target, "--page", "1"], + ["grep", query, target], + ] + for command in commands: + print(f"\n$ pifs {' '.join(command)}") + result = subprocess.run( + ["pifs", "--workspace", str(workspace), *command], + check=False, + ) + if result.returncode: + return result.returncode + return 0 + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("document", type=Path, help="PDF or Markdown document") + parser.add_argument("--workspace", type=Path, default=Path("pifs-workspace")) + parser.add_argument("--query", default="key findings") + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + raise SystemExit(run(args.workspace, args.document, args.query)) diff --git a/pageindex/__init__.py b/pageindex/__init__.py index 658003bf5..c3fb0b0ae 100644 --- a/pageindex/__init__.py +++ b/pageindex/__init__.py @@ -1,4 +1,22 @@ -from .page_index import * -from .page_index_md import md_to_tree -from .retrieve import get_document, get_document_structure, get_page_content -from .client import PageIndexClient +import os + +os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "true") + +_OPTIONAL_CORE_IMPORTS = {"litellm", "openai", "PyPDF2", "pymupdf"} + +try: + from .page_index import * + from .page_index_md import md_to_tree + from .retrieve import get_document, get_document_structure, get_page_content + from .client import PageIndexClient +except ModuleNotFoundError as exc: + if exc.name not in _OPTIONAL_CORE_IMPORTS: + raise + + +def __getattr__(name: str): + if name == "PageIndexFileSystem": + from .filesystem import PageIndexFileSystem + + return PageIndexFileSystem + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/pageindex/config.yaml b/pageindex/config.yaml index 591fe9331..5da9ee461 100644 --- a/pageindex/config.yaml +++ b/pageindex/config.yaml @@ -1,4 +1,4 @@ -model: "gpt-4o-2024-11-20" +model: "gpt-5.4" # model: "anthropic/claude-sonnet-4-6" retrieve_model: "gpt-5.4" # defaults to `model` if not set toc_check_page_num: 20 @@ -7,4 +7,4 @@ max_token_num_each_node: 20000 if_add_node_id: "yes" if_add_node_summary: "yes" if_add_doc_description: "no" -if_add_node_text: "no" \ No newline at end of file +if_add_node_text: "no" diff --git a/pageindex/filesystem/__init__.py b/pageindex/filesystem/__init__.py new file mode 100644 index 000000000..e92b682d2 --- /dev/null +++ b/pageindex/filesystem/__init__.py @@ -0,0 +1,3 @@ +from .core import PageIndexFileSystem + +__all__ = ["PageIndexFileSystem"] diff --git a/pageindex/filesystem/_embedding_identity.py b/pageindex/filesystem/_embedding_identity.py new file mode 100644 index 000000000..a55445f8d --- /dev/null +++ b/pageindex/filesystem/_embedding_identity.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +from urllib.parse import urlsplit, urlunsplit + + +DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1" + + +def normalize_base_url(value: str | None) -> str: + raw = str(value or DEFAULT_OPENAI_BASE_URL).strip() + parsed = urlsplit(raw) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValueError("embedding base URL must be an absolute HTTP(S) URL") + if parsed.username or parsed.password or parsed.query or parsed.fragment: + raise ValueError( + "embedding base URL must not contain credentials, query parameters, or a fragment" + ) + return urlunsplit( + ( + parsed.scheme.lower(), + parsed.netloc.lower(), + parsed.path.rstrip("/"), + "", + "", + ) + ) + + +def normalize_model(value: str | None) -> str: + model = str(value or "").strip() + if not model: + raise ValueError("embedding model must not be empty") + return model diff --git a/pageindex/filesystem/_projection_topology.py b/pageindex/filesystem/_projection_topology.py new file mode 100644 index 000000000..8c209548c --- /dev/null +++ b/pageindex/filesystem/_projection_topology.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from pathlib import Path + + +def projection_database_paths(index_dir: str | Path) -> tuple[Path, Path]: + index_dir = Path(index_dir).expanduser() + return index_dir / "summary.sqlite", index_dir / "embedding_cache.sqlite" + + +def projection_database_path_present(path: str | Path) -> bool: + path = Path(path) + return path.exists() or path.is_symlink() + + +def projection_database_pair( + index_dir: str | Path, +) -> tuple[Path, Path] | None: + summary_path, cache_path = projection_database_paths(index_dir) + summary_exists = projection_database_path_present(summary_path) + cache_exists = projection_database_path_present(cache_path) + if not summary_exists and not cache_exists: + return None + if not summary_exists or not cache_exists: + raise RuntimeError( + "PIFS Summary Projection topology is incomplete; migrate this workspace with " + "pifs-data/scripts/migrate_pifs_workspace.py before opening it." + ) + return summary_path, cache_path diff --git a/pageindex/filesystem/_sqlite_schema.py b/pageindex/filesystem/_sqlite_schema.py new file mode 100644 index 000000000..691184391 --- /dev/null +++ b/pageindex/filesystem/_sqlite_schema.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import sqlite3 +from typing import Any + + +def regular_table_names(connection: sqlite3.Connection) -> set[str]: + return { + str(row[0]) + for row in connection.execute( + "SELECT name FROM sqlite_master " + "WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" + ) + } + + +def sqlite_schema_signature( + connection: sqlite3.Connection, + tables: set[str], +) -> dict[str, Any]: + columns = { + table: tuple( + tuple(row) + for row in connection.execute(f'PRAGMA table_xinfo("{table}")') + ) + for table in sorted(tables) + } + indexes: dict[str, tuple[tuple[Any, ...], ...]] = {} + foreign_keys: dict[str, tuple[tuple[Any, ...], ...]] = {} + for table in sorted(tables): + table_indexes = [] + for row in connection.execute(f'PRAGMA index_list("{table}")'): + name = str(row[1]) + origin = str(row[3]) + index_columns = tuple( + str(column[2]) + for column in connection.execute(f'PRAGMA index_info("{name}")') + ) + table_indexes.append( + ( + name if origin == "c" else None, + int(row[2]), + origin, + int(row[4]), + index_columns, + ) + ) + indexes[table] = tuple(sorted(table_indexes, key=repr)) + foreign_keys[table] = tuple( + sorted( + ( + str(row[2]), + str(row[3]), + str(row[4]), + str(row[5]), + str(row[6]), + str(row[7]), + ) + for row in connection.execute(f'PRAGMA foreign_key_list("{table}")') + ) + ) + return { + "tables": tables, + "columns": columns, + "indexes": indexes, + "foreign_keys": foreign_keys, + } + + +def normalized_table_sql(connection: sqlite3.Connection, table: str) -> str: + row = connection.execute( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?", + (table,), + ).fetchone() + return " ".join(str(row[0] if row else "").lower().split()) diff --git a/pageindex/filesystem/_workspace_consistency.py b/pageindex/filesystem/_workspace_consistency.py new file mode 100644 index 000000000..41fbb412a --- /dev/null +++ b/pageindex/filesystem/_workspace_consistency.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +import sqlite3 +from pathlib import Path + + +def _readonly_connection(path: Path) -> sqlite3.Connection: + connection = sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True) + connection.row_factory = sqlite3.Row + return connection + + +def _inconsistent_workspace_error(detail: str) -> RuntimeError: + return RuntimeError( + f"Inconsistent PIFS workspace: {detail}; migrate this workspace with " + "pifs-data/scripts/migrate_pifs_workspace.py before opening it." + ) + + +def validate_catalog_root(catalog_path: str | Path) -> None: + try: + with _readonly_connection(Path(catalog_path)) as catalog: + root = catalog.execute( + "SELECT folder_id, parent_id, name FROM folders WHERE path = '/'" + ).fetchone() + except sqlite3.Error as exc: + raise _inconsistent_workspace_error( + "the catalog canonical root folder could not be read" + ) from exc + if ( + root is None + or root["folder_id"] != "folder_root" + or root["parent_id"] is not None + or root["name"] != "/" + ): + raise _inconsistent_workspace_error( + "the catalog is missing its canonical root folder" + ) + + +def validate_workspace_consistency( + catalog_path: str | Path, + summary_path: str | Path, +) -> None: + catalog_path = Path(catalog_path) + validate_catalog_root(catalog_path) + try: + with _readonly_connection(catalog_path) as catalog: + active_file_refs = { + str(row[0]) + for row in catalog.execute( + "SELECT file_ref FROM files WHERE deleted_at IS NULL" + ) + } + with _readonly_connection(Path(summary_path)) as summary: + projected_file_refs = { + str(row[0]) + for row in summary.execute( + "SELECT file_ref FROM semantic_index_docs" + ) + } + except sqlite3.Error as exc: + raise _inconsistent_workspace_error( + "catalog and Summary Projection references could not be read" + ) from exc + orphaned = sorted(projected_file_refs - active_file_refs) + if orphaned: + shown = ", ".join(orphaned[:5]) + raise _inconsistent_workspace_error( + "Summary Projection file_ref values do not reference active catalog files: " + f"{shown}" + ) + missing = sorted(active_file_refs - projected_file_refs) + if missing: + shown = ", ".join(missing[:5]) + raise _inconsistent_workspace_error( + "active catalog file_ref values are missing from the Summary Projection: " + f"{shown}" + ) + + +def validate_catalog_without_projection(catalog_path: str | Path) -> None: + catalog_path = Path(catalog_path) + validate_catalog_root(catalog_path) + try: + with _readonly_connection(catalog_path) as catalog: + active_file_refs = sorted( + str(row[0]) + for row in catalog.execute( + "SELECT file_ref FROM files WHERE deleted_at IS NULL" + ) + ) + except sqlite3.Error as exc: + raise _inconsistent_workspace_error( + "active catalog references could not be read" + ) from exc + if active_file_refs: + shown = ", ".join(active_file_refs[:5]) + raise _inconsistent_workspace_error( + "active catalog files require a complete Summary Projection: " + f"{shown}" + ) diff --git a/pageindex/filesystem/agent.py b/pageindex/filesystem/agent.py new file mode 100644 index 000000000..fd4176d6b --- /dev/null +++ b/pageindex/filesystem/agent.py @@ -0,0 +1,615 @@ +from __future__ import annotations + +import asyncio +import concurrent.futures +import json +import os +import re +import sys +import time +from dataclasses import asdict, is_dataclass +from typing import Any, Mapping, TextIO + +from .commands import PIFSCommandExecutor +from .core import PageIndexFileSystem + + +TRUTHY_ENV_VALUES = {"1", "true", "yes", "on"} +PIFS_AGENT_TRACING_ENV = "PAGEINDEX_PIFS_AGENT_TRACING" +PIFS_AGENT_RAW_REASONING_ENV = "PAGEINDEX_PIFS_AGENT_RAW_REASONING" + +AGENT_SYSTEM_PROMPT = """ +You are the PageIndex FileSystem Demo Agent, developed by the VectifyAI Team. +Your job is to answer questions about the caller's current PageIndex FileSystem +workspace. + +You can inspect the corpus only by calling the bash tool. The bash tool is a +read-only PageIndex virtual shell, not a real operating-system shell. + +If the user asks who you are, answer with this identity and mention that you can +help inspect and answer questions about the current PIFS workspace. If the user +asks a general question unrelated to the current workspace, do not answer it as +a general-purpose assistant; briefly say that you can only answer workspace- +related questions and invite them to ask about files, folders, metadata, or +document contents in the workspace. + +If the user asks what tools or capabilities you have, describe only the PIFS +virtual shell capabilities available inside this workspace: tree, browse, +stat, cat, and grep. Describe PIFS Listing as tree -L 1. Do not mention +host runtime tools, SDK internals, or orchestration helpers that are not part of +the PIFS shell. + +If the user asks a workspace-related topic question without naming a specific +file, treat it as a retrieval task. Start with tree to understand folder +structure, choose a folder, then use browse with the user's topic as the query +to find candidate files. Inspect evidence before answering. Ask the user to +clarify only after the persistence protocol cannot identify relevant evidence. + +Follow the task prompt for command policy, retrieval strategy, and answer +format. If the caller needs stricter behavior, pass an explicit system_prompt. +""" + +BASH_TOOL_DESCRIPTION = """ +Run a command in the PageIndex FileSystem virtual shell. This is not a real +operating-system shell. The tool is read-only and always returns JSON. +Use only: tree, browse, stat, cat, and grep. +Start broad workspace questions with tree. Then choose a physical folder, +inspect metadata-derived virtual axes such as @year or @ticker with tree, and +select exact metadata values through paths such as @field/value. Copy returned +scope paths into later commands. Run browse "" for document +discovery. Use browse -R when low-signal folder or virtual-node labels do not +help prune the query, or after scoped browse misses/rephrasing. Browse returns document +candidates only, not final evidence. Use stat only for identity, status, or +metadata. Use cat --structure before any cat --page N[-M] for +that file. Use grep only as a single-document lexical fallback. +Other shell commands, pipes, stat modes, folder-wide lexical searches, and +implicit cat reads are unsupported. +""" + +AGENT_TOOL_POLICY = """ +Tool policy: +- The bash tool is a PageIndex virtual shell, not an operating-system shell. +- The default agent tool surface is read-only. +- Use only tree, browse, stat, cat, and grep. PIFS Listing is tree -L 1. +- Folder paths such as /documents are positional command targets; never put folder paths in --where. +- Start with tree to understand workspace and folder structure before document retrieval. +- After choosing a folder, use tree to inspect metadata-derived virtual axis nodes such as @year, then tree /@field to inspect @field/value nodes, and copy returned scope paths instead of handwriting encoded values. +- After choosing a folder or scope, use browse "" for relevance-ranked document candidates; quote multi-word queries, for example browse /documents "Federal Reserve". +- Use browse -R from a broader folder when low-signal folder or virtual-node labels do not help prune the query. +- If browse misses: inspect sibling or child folders with tree, browse another plausible folder, rephrase the query and browse again, then use browse -R from a broader folder. +- Only after that persistence protocol may you say the workspace lacks evidence. +- browse always uses summary retrieval. +- Use metadata scope paths for exact filters. Keep --where as a fallback for OR, IN, or contains only when tree paths cannot express the predicate. +- browse candidates are not final evidence. After selecting candidates, verify the relevant facts with cat or grep before making source-backed claims. +- For financial statement, ratio, or calculation questions, prefer citing a primary statement, reconciliation table, or official table when available; use summaries, MD&A, or footnotes as supporting evidence. +- Use grep for a selected single file only. +- Other shell commands, pipes, schema browsing, batch metadata commands, folder-wide lexical searches, implicit cat reads, and general-knowledge fallback are unsupported. +- Command errors are JSON; recover by trying an available command. +- Use cat or grep to gather evidence before making source-backed claims. +- Do not reconstruct a file path from a title. Use the exact path returned by PIFS commands and quote paths that contain spaces. +- For broad topic, method, or "what solution" questions that are likely about the workspace, browse for candidate documents before asking the user to choose a document. +- Use stat only for identity, metadata, or status questions. Do not run stat merely to understand what a document says. +- Prefer target-first cat syntax with stable targets: cat --structure, cat --page 31-59. +- cat --structure returns the cached PageIndex structure JSON without text fields. +- For PDF/PageIndex document Q&A, run cat --structure before the first cat --page call for that target; page reads without structure are blind page guessing. +- cat --page accepts at most 5 pages at once. If a larger range is needed, first inspect cat --structure and then read a smaller page range. +- When recovering from cat page/text limit errors, stop if the evidence is sufficient; if it is not sufficient, make another smaller call before answering. +- After cat --structure identifies a relevant section/subsection, use cat --page - for exact evidence. +- Use cat --page - when the user explicitly asks for pages/page ranges or when you need exact page text to verify evidence. +- Do not guess cat --page ranges from grep line numbers, text offsets, or table-of-contents intuition; use cat --structure to map the document first. +- Avoid fetching a broad page span unless page-level citation or verification is required. +- Do not call cat --page ; if you need a page span, use cat --page -. +- For metadata or summary-field questions, run stat for relevant files before answering. +- Distinguish generated metadata from caller-provided custom metadata when the evidence supports it. +- Do a final consistency check before answering: the headline number, calculation, unit, sign, and yes/no polarity must agree with each other. +""" + +STREAM_MODE_ALIASES = { + "": "off", + "none": "off", + "false": "off", + "0": "off", + "off": "off", + "tool": "tools", + "tools": "tools", + "model": "model", + "output": "model", + "outputs": "model", + "think": "model", + "all": "all", + "debug": "all", +} +AGENT_STREAM_MODE_CHOICES = sorted(item for item in STREAM_MODE_ALIASES if item) +REASONING_EFFORT_CHOICES = ["none", "minimal", "low", "medium", "high", "xhigh"] +REASONING_SUMMARY_CHOICES = ["none", "auto", "concise", "detailed"] + + +def should_use_openai_compatible_chat_model(base_url: str | None) -> bool: + if not base_url: + return False + normalized = base_url.strip().rstrip("/") + return normalized not in {"https://api.openai.com", "https://api.openai.com/v1"} + + +def env_flag_enabled(name: str, environ: Mapping[str, str] | None = None) -> bool: + source = os.environ if environ is None else environ + value = source.get(name, "") + return value.strip().lower() in TRUTHY_ENV_VALUES + + +def pifs_agent_tracing_enabled(environ: Mapping[str, str] | None = None) -> bool: + return env_flag_enabled(PIFS_AGENT_TRACING_ENV, environ) + + +def should_disable_pifs_agent_tracing(environ: Mapping[str, str] | None = None) -> bool: + return not pifs_agent_tracing_enabled(environ) + + +def pifs_agent_raw_reasoning_enabled(environ: Mapping[str, str] | None = None) -> bool: + return env_flag_enabled(PIFS_AGENT_RAW_REASONING_ENV, environ) + + +def normalize_reasoning_effort(reasoning_effort: str | None) -> str | None: + if reasoning_effort is None or not reasoning_effort.strip(): + return None + effort = reasoning_effort.strip().lower() + if effort not in REASONING_EFFORT_CHOICES: + allowed = ", ".join(REASONING_EFFORT_CHOICES) + raise ValueError(f"Unknown reasoning effort: {reasoning_effort!r}. Allowed: {allowed}") + return effort + + +def normalize_reasoning_summary(reasoning_summary: str | None) -> str | None: + if reasoning_summary is None or not reasoning_summary.strip(): + return None + summary = reasoning_summary.strip().lower() + if summary not in REASONING_SUMMARY_CHOICES: + allowed = ", ".join(REASONING_SUMMARY_CHOICES) + raise ValueError(f"Unknown reasoning summary: {reasoning_summary!r}. Allowed: {allowed}") + return None if summary == "none" else summary + + +def build_agent_model_settings( + *, + reasoning_effort: str | None = None, + reasoning_summary: str | None = None, +) -> Any | None: + effort = normalize_reasoning_effort(reasoning_effort) + summary = normalize_reasoning_summary(reasoning_summary) + if effort is None and summary is None: + return None + if effort not in {None, "none"} and summary is None: + summary = "auto" + + from agents import ModelSettings + from openai.types.shared import Reasoning + + reasoning_kwargs = {} + if effort is not None: + reasoning_kwargs["effort"] = effort + if summary is not None: + reasoning_kwargs["summary"] = summary + return ModelSettings(reasoning=Reasoning(**reasoning_kwargs), verbosity="low") + + +def normalize_agent_stream_mode(stream_mode: str | None) -> str: + mode = STREAM_MODE_ALIASES.get((stream_mode or "off").strip().lower()) + if mode is None: + allowed = ", ".join(sorted({"off", "tools", "model", "all"})) + raise ValueError(f"Unknown PIFS agent stream mode: {stream_mode!r}. Allowed: {allowed}") + return mode + + +def serialize_agent_final_output(value: Any) -> str: + if value is None: + return "" + if isinstance(value, str): + return value + if hasattr(value, "model_dump_json"): + return value.model_dump_json() + if is_dataclass(value): + return json.dumps(asdict(value), ensure_ascii=False) + if isinstance(value, (dict, list)): + return json.dumps(value, ensure_ascii=False) + return str(value) + + +def compact_tool_output_preview( + output: str, + *, + preview_chars: int = 700, + max_lines: int = 8, +) -> str: + cleaned = str(output).replace("\r", "\n").replace("\f", "\n") + cleaned = "".join( + ch if ch == "\n" or ch == "\t" or ord(ch) >= 32 else " " + for ch in cleaned + ) + lines = [ + re.sub(r"[ \t]{2,}", " ", line).strip() + for line in cleaned.splitlines() + if line.strip() + ] + is_large_result = len(cleaned) > preview_chars or len(lines) > max_lines + preview = "\n".join(lines[:max_lines]) + if len(preview) > preview_chars: + preview = preview[:preview_chars].rstrip() + "..." + omitted = len(lines) - min(len(lines), max_lines) + if is_large_result: + preview = f"[large PIFS result: {len(cleaned)} chars; showing compact preview]\n" + preview + if omitted > 0: + preview += f"\n... [{omitted} more lines omitted from preview]" + if len(cleaned) > preview_chars: + preview += "\n... [full result returned to agent; terminal preview shortened]" + return preview + + +def build_agent_initial_context( + filesystem: PageIndexFileSystem, + *, + root: str = "/", + executor: PIFSCommandExecutor | None = None, +) -> str: + executor = executor or PIFSCommandExecutor(filesystem) + return "\n".join( + [ + f"Root path: {root}", + "Top-level tree:", + executor.execute(f"tree {root} -L 1"), + "Workspace retrieval capabilities:", + executor.describe_available_command_surfaces(), + ] + ) + + +def build_pifs_agent_instructions( + filesystem: PageIndexFileSystem, + *, + root: str = "/", + system_prompt: str | None = None, + executor: PIFSCommandExecutor | None = None, +) -> str: + initial_context = build_agent_initial_context( + filesystem, + root=root, + executor=executor, + ) + return "\n\n".join( + [ + (system_prompt or AGENT_SYSTEM_PROMPT).strip(), + AGENT_TOOL_POLICY.strip(), + "Workspace context:\n" + initial_context, + ] + ) + + +class PIFSAgentStreamObserver: + def __init__( + self, + stream_mode: str, + *, + stream_log: list[dict[str, Any]] | None = None, + output: TextIO | None = None, + include_raw_reasoning: bool | None = None, + ) -> None: + self.stream_mode = normalize_agent_stream_mode(stream_mode) + self.stream_log = stream_log + self.output = output or sys.stdout + self.include_raw_reasoning = ( + pifs_agent_raw_reasoning_enabled() + if include_raw_reasoning is None + else include_raw_reasoning + ) + self._printed_section: str | None = None + self._buffers: dict[str, list[str]] = { + "output": [], + "think": [], + "think_summary": [], + "tool_args": [], + } + + @property + def wants_model_stream(self) -> bool: + return self.stream_mode in {"model", "all"} + + @property + def wants_tool_stream(self) -> bool: + return self.stream_mode in {"tools", "all"} + + @property + def has_output_text(self) -> bool: + return bool(self._buffers["output"]) + + def handle_event(self, event: Any) -> None: + if getattr(event, "type", None) == "raw_response_event": + self._handle_raw_response_event(getattr(event, "data", None)) + elif getattr(event, "type", None) == "run_item_stream_event": + self._handle_run_item_event(event) + + def finish(self, final_output: Any = None) -> None: + if self.wants_model_stream and not self.has_output_text and final_output: + self._emit("output", str(final_output), "[llm final output stream]") + if self._printed_section is not None: + print(file=self.output, flush=True) + self._printed_section = None + if self.stream_log is not None: + for kind, parts in self._buffers.items(): + text = "".join(parts) + if text: + self.stream_log.append({"kind": kind, "text": text}) + + def _handle_raw_response_event(self, data: Any) -> None: + event_type = getattr(data, "type", "") + delta = getattr(data, "delta", None) + if not isinstance(delta, str) or not delta: + return + if event_type == "response.output_text.delta": + self._emit("output", delta, "[llm final output stream]") + elif event_type == "response.reasoning_text.delta": + if self.include_raw_reasoning: + self._emit("think", delta, "[llm reasoning text stream]") + elif event_type == "response.reasoning_summary_text.delta": + self._emit("think_summary", delta, "[llm reasoning summary stream]") + elif event_type == "response.function_call_arguments.delta": + self._buffers["tool_args"].append(delta) + + def _handle_run_item_event(self, event: Any) -> None: + name = getattr(event, "name", "") + item = getattr(event, "item", None) + item_type = getattr(item, "type", "") + if self.stream_log is not None and name in {"message_output_created", "reasoning_item_created"}: + self.stream_log.append({"kind": "run_item", "name": name, "item_type": item_type}) + + def _emit(self, kind: str, text: str, label: str) -> None: + if kind == "tool_args": + should_print = self.wants_tool_stream + else: + should_print = self.wants_model_stream + if not should_print: + return + self._buffers[kind].append(text) + if self._printed_section != kind: + if self._printed_section is not None: + print(file=self.output, flush=True) + print(f"\n{label}", file=self.output, flush=True) + self._printed_section = kind + print(text, end="", file=self.output, flush=True) + + def emit_tool_call(self, command: str, *, force: bool = False) -> None: + if not command.strip(): + return + if self.stream_log is not None: + self.stream_log.append({"kind": "tool_call", "command": command}) + if not (force or self.wants_tool_stream): + return + self._start_section("tool_call", "[llm -> pifs command]") + print(command, file=self.output, flush=True) + + def emit_tool_result( + self, + *, + ok: bool, + output: str, + seconds: float, + force: bool = False, + preview_chars: int = 1000, + ) -> None: + if self.stream_log is not None: + self.stream_log.append( + { + "kind": "tool_result", + "ok": ok, + "seconds": round(seconds, 4), + "output_chars": len(output), + "preview": compact_tool_output_preview(output, preview_chars=preview_chars), + } + ) + if not (force or self.wants_tool_stream): + return + preview = compact_tool_output_preview(output, preview_chars=preview_chars) + self._start_section("tool_result", "[pifs -> llm result preview]") + print( + f"ok={str(ok).lower()} seconds={seconds:.4f} output_chars={len(output)}", + file=self.output, + flush=True, + ) + print(preview, file=self.output, flush=True) + + def _start_section(self, kind: str, label: str) -> None: + if self._printed_section is not None: + print(file=self.output, flush=True) + print(f"\n{label}", file=self.output, flush=True) + self._printed_section = kind + + +def run_pifs_agent( + filesystem: PageIndexFileSystem, + question: str, + *, + model: str, + root: str = "/", + system_prompt: str | None = None, + max_turns: int = 20, + max_seconds: float | None = 60, + verbose: bool = False, + stream_mode: str = "off", + reasoning_effort: str | None = None, + reasoning_summary: str | None = None, + output_type: type[Any] | None = None, + tool_log: list[dict[str, Any]] | None = None, + agent_log: list[dict[str, Any]] | None = None, +) -> str: + session = PIFSAgentSession( + filesystem, + model=model, + root=root, + system_prompt=system_prompt, + max_turns=max_turns, + max_seconds=max_seconds, + verbose=verbose, + stream_mode=stream_mode, + reasoning_effort=reasoning_effort, + reasoning_summary=reasoning_summary, + output_type=output_type, + tool_log=tool_log, + agent_log=agent_log, + persist_conversation=False, + ) + return session.run(question) + + +class PIFSAgentSession: + def __init__( + self, + filesystem: PageIndexFileSystem, + *, + model: str, + root: str = "/", + system_prompt: str | None = None, + max_turns: int = 20, + max_seconds: float | None = 60, + verbose: bool = False, + stream_mode: str = "off", + reasoning_effort: str | None = None, + reasoning_summary: str | None = None, + output_type: type[Any] | None = None, + tool_log: list[dict[str, Any]] | None = None, + agent_log: list[dict[str, Any]] | None = None, + persist_conversation: bool = True, + ) -> None: + self.filesystem = filesystem + self.max_turns = max_turns + self.max_seconds = max_seconds + self.verbose = verbose + self.tool_log = tool_log + self.agent_log = agent_log + self.normalized_stream_mode = normalize_agent_stream_mode(stream_mode) + self.observer: PIFSAgentStreamObserver | None = None + + try: + from agents import ( + Agent, + OpenAIChatCompletionsModel, + function_tool, + set_tracing_disabled, + ) + from agents.memory import SQLiteSession + from openai import AsyncOpenAI + except ModuleNotFoundError as exc: + if exc.name == "agents": + raise RuntimeError( + "openai-agents is required to run the PageIndex FileSystem agent" + ) from exc + raise + + set_tracing_disabled(should_disable_pifs_agent_tracing()) + self.executor = PIFSCommandExecutor(filesystem) + instructions = build_pifs_agent_instructions( + filesystem, + root=root, + system_prompt=system_prompt, + executor=self.executor, + ) + + @function_tool(description_override=BASH_TOOL_DESCRIPTION.strip()) + def bash(command: str) -> str: + """Run an allowed PageIndex FileSystem virtual shell command.""" + return self._run_bash(command) + + model_settings = build_agent_model_settings( + reasoning_effort=reasoning_effort, + reasoning_summary=reasoning_summary, + ) + base_url = os.environ.get("OPENAI_BASE_URL") + model_config = model + if should_use_openai_compatible_chat_model(base_url): + model_config = OpenAIChatCompletionsModel( + model=model, + openai_client=AsyncOpenAI( + api_key=os.environ.get("OPENAI_API_KEY"), + base_url=base_url, + ), + ) + + agent_kwargs: dict[str, Any] = { + "name": "PageIndexFileSystem", + "instructions": instructions, + "tools": [bash], + "model": model_config, + } + if model_settings is not None: + agent_kwargs["model_settings"] = model_settings + if output_type is not None: + agent_kwargs["output_type"] = output_type + self.agent = Agent(**agent_kwargs) + self.session = SQLiteSession("pifs-chat") if persist_conversation else None + + def run(self, question: str) -> str: + self.observer = PIFSAgentStreamObserver( + self.normalized_stream_mode, + stream_log=self.agent_log, + ) + + async def _run_streamed() -> str: + from agents import Runner + + streamed_run = Runner.run_streamed( + self.agent, + question, + max_turns=self.max_turns, + session=self.session, + ) + final_output = "" + try: + async for event in streamed_run.stream_events(): + self.observer.handle_event(event) + final_output = serialize_agent_final_output(streamed_run.final_output) + return final_output + finally: + if not final_output and streamed_run.final_output: + final_output = serialize_agent_final_output(streamed_run.final_output) + self.observer.finish(final_output) + + async def _run() -> str: + if self.max_seconds is None or self.max_seconds <= 0: + return await _run_streamed() + try: + return await asyncio.wait_for(_run_streamed(), timeout=self.max_seconds) + except asyncio.TimeoutError as exc: + raise TimeoutError(f"MaxSecondsExceeded: exceeded {self.max_seconds:g}s") from exc + + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(_run()) + + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as pool: + return pool.submit(asyncio.run, _run()).result() + + def _run_bash(self, command: str) -> str: + started = time.time() + assert self.observer is not None + self.observer.emit_tool_call(command, force=self.verbose) + output = self.executor.execute(command) + try: + ok = bool(json.loads(output).get("success")) + except json.JSONDecodeError: + ok = False + seconds = time.time() - started + if self.tool_log is not None: + self.tool_log.append( + { + "command": command, + "ok": ok, + "seconds": round(seconds, 4), + "output_chars": len(output), + "preview": output[:500], + } + ) + self.observer.emit_tool_result( + ok=ok, + output=output, + seconds=seconds, + force=self.verbose, + ) + return output diff --git a/pageindex/filesystem/cli.py b/pageindex/filesystem/cli.py new file mode 100644 index 000000000..8b03a65ba --- /dev/null +++ b/pageindex/filesystem/cli.py @@ -0,0 +1,472 @@ +from __future__ import annotations + +import argparse +import contextlib +import json +import os +import re +import shlex +import sys +from pathlib import Path +from typing import Iterator, TextIO + +from .agent import ( + PIFSAgentSession, + REASONING_EFFORT_CHOICES, + REASONING_SUMMARY_CHOICES, + run_pifs_agent, +) +from .commands import PIFSCommandError, PIFSCommandExecutor +from .core import PageIndexFileSystem + + +AGENT_STREAM_MODE_CHOICES = ("off", "tools", "model", "all") +DEFAULT_AGENT_MODEL = "gpt-5.4" +EXIT_COMMANDS = {"exit", "quit", ":q"} +ANSI_ESCAPE_RE = re.compile(r"\x1b(?:\[[0-?]*[ -/]*[@-~]|.)") +PIFS_CONFIG_FILE_ENV = "PIFS_CONFIG_FILE" +PIFS_WORKSPACE_ENV = "PIFS_WORKSPACE" +PERSISTED_CONFIG_KEYS = { + "workspace", + "embedding_api_key", + "embedding_base_url", + "embedding_model", + "embedding_dimensions", + "embedding_timeout", +} + + +def _config_path() -> Path: + override = os.environ.get(PIFS_CONFIG_FILE_ENV) + if override: + return Path(override).expanduser() + config_home = os.environ.get("XDG_CONFIG_HOME") + root = Path(config_home).expanduser() if config_home else Path.home() / ".config" + return root / "pageindex" / "pifs.json" + + +def _read_config() -> dict[str, str]: + path = _config_path() + if not path.exists(): + return {} + with path.open("r", encoding="utf-8") as handle: + payload = json.load(handle) + if not isinstance(payload, dict): + raise ValueError(f"invalid PIFS config file: {path}") + return { + str(key): str(value) + for key, value in payload.items() + if key in PERSISTED_CONFIG_KEYS and value is not None + } + + +def _write_config(config: dict[str, str]) -> Path: + path = _config_path() + path.parent.mkdir(parents=True, exist_ok=True) + persisted = { + key: value + for key, value in config.items() + if key in PERSISTED_CONFIG_KEYS and value is not None + } + with path.open("w", encoding="utf-8") as handle: + json.dump(persisted, handle, indent=2, sort_keys=True) + handle.write("\n") + return path + + +def _configured_workspace() -> str | None: + return _read_config().get("workspace") + + +def _resolve_workspace(value: str | None) -> str | None: + return value or os.environ.get(PIFS_WORKSPACE_ENV) or _configured_workspace() + + +def _load_env_file(path: str | None = None, *, workspace: str | None = None) -> Path | None: + from dotenv import load_dotenv + + if path: + env_path = Path(path).expanduser() + if not env_path.exists(): + raise FileNotFoundError(f"env file not found: {env_path}") + load_dotenv(env_path, override=True) + return env_path + + env_override = os.environ.get("PIFS_ENV_FILE") + if env_override: + return _load_env_file(env_override) + + starts = [Path.cwd()] + if workspace: + starts.append(Path(workspace).expanduser()) + seen: set[Path] = set() + for start in starts: + current = start.resolve() if start.exists() else start.resolve(strict=False) + if current.is_file(): + current = current.parent + for parent in (current, *current.parents): + candidate = parent / ".env" + if candidate in seen: + continue + seen.add(candidate) + if candidate.exists(): + load_dotenv(candidate, override=False) + return candidate + return None + + +def _agent_model_default() -> str: + return ( + os.environ.get("PIFS_AGENT_MODEL") + or os.environ.get("PIFS_MODEL") + or DEFAULT_AGENT_MODEL + ) + + +def _add_agent_arguments( + parser: argparse.ArgumentParser, + *, + workspace_default: str | None, + default_stream_mode: str, +) -> None: + parser.add_argument("--workspace", default=workspace_default) + parser.add_argument("--env-file", default=None) + parser.add_argument("--model", default=_agent_model_default()) + parser.add_argument( + "--stream-mode", + default=default_stream_mode, + choices=AGENT_STREAM_MODE_CHOICES, + ) + parser.add_argument("--max-turns", type=int, default=20) + parser.add_argument("--max-seconds", type=float, default=60) + parser.add_argument( + "--reasoning-effort", + choices=REASONING_EFFORT_CHOICES, + default=None, + ) + parser.add_argument( + "--reasoning-summary", + choices=REASONING_SUMMARY_CHOICES, + default=None, + ) + + +def _parse_agent_command( + command_name: str, + argv: list[str], + *, + workspace_default: str | None, + default_stream_mode: str, +) -> argparse.Namespace: + parser = argparse.ArgumentParser( + prog=f"pifs {command_name}", + description=f"PageIndex FileSystem {command_name}", + ) + _add_agent_arguments( + parser, + workspace_default=workspace_default, + default_stream_mode=default_stream_mode, + ) + if command_name == "ask": + parser.add_argument("question", nargs=argparse.REMAINDER) + args = parser.parse_args(argv) + _load_env_file(args.env_file, workspace=args.workspace) + args.workspace = _resolve_workspace(args.workspace) + if not args.workspace: + parser.error("--workspace is required unless PIFS_WORKSPACE is set or `pifs set workspace ` has been run") + return args + + +def _optional_int(name: str, value: str | None) -> int | None: + if value is None or not value.strip(): + return None + try: + return int(value) + except ValueError as exc: + raise ValueError(f"{name} must be an integer") from exc + + +def _optional_float(name: str, value: str | None) -> float | None: + if value is None or not value.strip(): + return None + try: + return float(value) + except ValueError as exc: + raise ValueError(f"{name} must be a number") from exc + + +def _filesystem_embedding_config() -> dict[str, object]: + config_values = _read_config() + config: dict[str, object] = {} + base_url = config_values.get("embedding_base_url") + api_key = ( + os.environ.get("PIFS_EMBEDDING_API_KEY") + or config_values.get("embedding_api_key") + or os.environ.get("OPENAI_API_KEY") + ) + model = config_values.get("embedding_model") + dimensions = _optional_int( + "embedding_dimensions", + config_values.get("embedding_dimensions"), + ) + timeout = _optional_float( + "embedding_timeout", + config_values.get("embedding_timeout"), + ) + if model and model.strip(): + config["summary_projection_embedding_model"] = model.strip() + if dimensions is not None: + config["summary_projection_embedding_dimensions"] = dimensions + if timeout is not None: + config["summary_projection_embedding_timeout"] = timeout + if base_url and base_url.strip(): + config["summary_projection_embedding_base_url"] = base_url.strip() + if api_key and api_key.strip(): + config["summary_projection_embedding_api_key"] = api_key.strip() + return config + + +def _filesystem_from_workspace(workspace: str) -> PageIndexFileSystem: + return PageIndexFileSystem( + Path(workspace).expanduser(), + **_filesystem_embedding_config(), + ) + + +def _agent_kwargs(args: argparse.Namespace) -> dict[str, object]: + return { + "model": args.model, + "stream_mode": args.stream_mode, + "max_turns": args.max_turns, + "max_seconds": args.max_seconds, + "reasoning_effort": args.reasoning_effort, + "reasoning_summary": args.reasoning_summary, + } + + +def _sanitize_chat_question(raw: str) -> str: + text = ANSI_ESCAPE_RE.sub("", raw) + chars: list[str] = [] + for char in text: + if char in {"\b", "\x7f"}: + if chars: + chars.pop() + continue + if char in {"\r", "\n"}: + continue + if ord(char) < 32 or ord(char) == 127: + continue + chars.append(char) + return "".join(chars).strip() + + +@contextlib.contextmanager +def _suppress_tty_input_echo(stdin: TextIO | None = None) -> Iterator[None]: + stream = sys.stdin if stdin is None else stdin + if not hasattr(stream, "isatty") or not stream.isatty(): + yield + return + try: + import termios + + fd = stream.fileno() + original = termios.tcgetattr(fd) + muted = original[:] + muted[3] = muted[3] & ~termios.ECHO + termios.tcsetattr(fd, termios.TCSADRAIN, muted) + except Exception: + yield + return + try: + yield + finally: + with contextlib.suppress(Exception): + termios.tcflush(fd, termios.TCIFLUSH) + with contextlib.suppress(Exception): + termios.tcsetattr(fd, termios.TCSADRAIN, original) + + +def _run_ask(argv: list[str], *, workspace_default: str | None) -> int: + args = _parse_agent_command( + "ask", + argv, + workspace_default=workspace_default, + default_stream_mode="off", + ) + question_tokens = [token for token in args.question if token != "--"] + question = " ".join(question_tokens).strip() + if not question: + raise ValueError("ask requires a question") + + filesystem = _filesystem_from_workspace(args.workspace) + answer = run_pifs_agent(filesystem, question, **_agent_kwargs(args)) + if args.stream_mode in {"off", "tools"}: + print(answer) + return 0 + + +def _run_chat(argv: list[str], *, workspace_default: str | None) -> int: + args = _parse_agent_command( + "chat", + argv, + workspace_default=workspace_default, + default_stream_mode="all", + ) + filesystem = _filesystem_from_workspace(args.workspace) + session = PIFSAgentSession(filesystem, **_agent_kwargs(args)) + while True: + try: + question = _sanitize_chat_question(input("pifs> ")) + except EOFError: + break + except KeyboardInterrupt: + print() + break + if not question: + continue + if question.lower() in EXIT_COMMANDS: + break + with _suppress_tty_input_echo(): + answer = session.run(question) + if args.stream_mode == "off": + print(answer) + return 0 + + +def _run_passthrough( + command_tokens: list[str], + *, + workspace: str, +) -> int: + filesystem = _filesystem_from_workspace(workspace) + executor = PIFSCommandExecutor(filesystem) + command = " ".join(shlex.quote(token) for token in command_tokens) + output = executor.execute(command) + print(output) + try: + payload = json.loads(output) + except json.JSONDecodeError: + return 0 + return 0 if payload.get("success") is not False else 2 + + +def _run_add(argv: list[str], *, workspace: str) -> int: + parser = argparse.ArgumentParser( + prog="pifs add", + description="Add a local file to a PageIndex FileSystem workspace", + ) + parser.add_argument("physical_path") + parser.add_argument("virtual_target") + args = parser.parse_args(argv) + + filesystem = _filesystem_from_workspace(workspace) + info = filesystem.add_file(args.physical_path, args.virtual_target) + print(f"added: {info.get('path')}") + return 0 + + +def _run_setmeta(argv: list[str], *, workspace: str) -> int: + parser = argparse.ArgumentParser( + prog="pifs setmeta", + description="Replace custom metadata for one PIFS document", + ) + parser.add_argument("--clear", action="store_true") + parser.add_argument("target") + parser.add_argument("metadata_json", nargs="?") + args = parser.parse_args(argv) + + if args.clear: + if args.metadata_json is not None: + parser.error("setmeta --clear accepts only a document target") + metadata = {} + else: + if args.metadata_json is None: + parser.error("setmeta requires a JSON object") + metadata = json.loads(args.metadata_json) + if not isinstance(metadata, dict): + parser.error("setmeta metadata must be a JSON object") + + filesystem = _filesystem_from_workspace(workspace) + info = filesystem.set_metadata(args.target, metadata, clear=args.clear) + document = { + "path": info.get("path"), + "document_id": info.get("external_id"), + "title": info.get("title"), + "status": info.get("pageindex_tree_status"), + "metadata": info.get("metadata", {}), + "metadata_status": info.get("metadata_status", {}), + } + print(json.dumps(document, ensure_ascii=False, sort_keys=True)) + return 0 + + +def _run_set(argv: list[str]) -> int: + parser = argparse.ArgumentParser( + prog="pifs set", + description="Set PageIndex FileSystem CLI defaults", + ) + parser.add_argument("name", choices=["workspace"]) + parser.add_argument("value") + args = parser.parse_args(argv) + + config = _read_config() + if args.name == "workspace": + workspace = Path(args.value).expanduser().resolve(strict=False) + config["workspace"] = str(workspace) + path = _write_config(config) + print(f"workspace: {workspace}") + print(f"config: {path}") + return 0 + raise ValueError(f"unknown config key: {args.name}") + + +def main(argv: list[str] | None = None) -> int: + argv = list(sys.argv[1:] if argv is None else argv) + _load_env_file() + parser = argparse.ArgumentParser(description="PageIndex FileSystem CLI") + parser.add_argument("--workspace", default=None) + parser.add_argument("--env-file", default=None) + parser.add_argument("command", nargs=argparse.REMAINDER) + args = parser.parse_args(argv) + _load_env_file(args.env_file, workspace=args.workspace) + args.workspace = _resolve_workspace(args.workspace) + + command_tokens = [token for token in args.command if token != "--"] + + if not command_tokens: + parser.error("a filesystem command is required") + + try: + command_name = command_tokens[0] + command_args = command_tokens[1:] + if command_name == "set": + return _run_set(command_args) + if command_name == "ask": + return _run_ask(command_args, workspace_default=args.workspace) + if command_name == "chat": + return _run_chat(command_args, workspace_default=args.workspace) + if command_name == "add": + if not args.workspace: + parser.error("--workspace is required unless PIFS_WORKSPACE is set or `pifs set workspace ` has been run") + return _run_add(command_args, workspace=args.workspace) + if command_name == "setmeta": + if not args.workspace: + parser.error("--workspace is required unless PIFS_WORKSPACE is set or `pifs set workspace ` has been run") + return _run_setmeta(command_args, workspace=args.workspace) + + if not args.workspace: + parser.error("--workspace is required unless PIFS_WORKSPACE is set or `pifs set workspace ` has been run") + return _run_passthrough( + command_tokens, + workspace=args.workspace, + ) + except PIFSCommandError as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + except Exception as exc: + print(f"ERROR: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pageindex/filesystem/commands.py b/pageindex/filesystem/commands.py new file mode 100644 index 000000000..22cfdd15e --- /dev/null +++ b/pageindex/filesystem/commands.py @@ -0,0 +1,602 @@ +from __future__ import annotations + +import json +import re +import shlex +from typing import Any + +from .core import PageIndexFileSystem + + +class PIFSCommandError(ValueError): + pass + + +class PIFSCommandExecutor: + COMMAND_NAMES = {"tree", "browse", "stat", "cat", "grep"} + FORBIDDEN_SUBSTRINGS = (";", "`", "$(", "||", "&&", "\n", "\r") + FORBIDDEN_TOKENS = {"|", ">", "<", ">>", "<<", "&"} + BROWSE_PAGE_SIZE = 10 + TREE_VALUE_PAGE_SIZE = 50 + MAX_TREE_FOLDERS = 200 + MAX_GREP_MATCHES = 20 + MAX_PAGE_SPAN = 5 + + def __init__(self, filesystem: PageIndexFileSystem): + self.filesystem = filesystem + + def allowed_commands(self) -> set[str]: + return set(self.COMMAND_NAMES) + + def describe_available_command_surfaces(self) -> str: + return "\n".join( + [ + "Available PIFS BashLike commands:", + "- tree [-L depth] [--page N]: folder and metadata-scope orientation", + '- browse "" [--page N] [--where JSON] [-R]: summary-ranked document discovery', + "- stat : single document identity, status, and metadata", + "- cat --structure | --page N[-M]: structure-first document reads", + "- grep : single-document lexical evidence fallback", + ] + ) + + def execute(self, command: str) -> str: + try: + data, next_steps = self._execute(command) + return self._success(data, next_steps=next_steps) + except PIFSCommandError as exc: + return self._error(str(exc)) + except (KeyError, ValueError) as exc: + return self._error(self._clean_error_message(exc)) + except Exception as exc: + return self._error(self._clean_error_message(exc)) + + def _execute(self, command: str) -> tuple[dict[str, Any], list[str]]: + self._validate_raw_command(command) + try: + tokens = shlex.split(command) + except ValueError as exc: + raise PIFSCommandError(f"Invalid command syntax: {exc}") from exc + if not tokens: + raise PIFSCommandError("Empty command") + self._validate_tokens(tokens) + if "--json" in tokens: + raise PIFSCommandError("--json is removed; command output is always JSON") + name, args = tokens[0], tokens[1:] + if name not in self.allowed_commands(): + raise PIFSCommandError(f"Unsupported command: {name}") + return getattr(self, f"_cmd_{name}")(args) + + def _cmd_tree(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: + path = "/" + depth = 10 + page = 1 + i = 0 + while i < len(args): + arg = args[i] + if arg == "-L": + i, value = self._option_value(args, i, "tree -L") + depth = self._parse_positive_int(value, "tree -L") + elif arg == "--page": + i, value = self._option_value(args, i, "tree --page") + page = self._parse_positive_int(value, "tree --page") + elif arg.startswith("-"): + raise PIFSCommandError(f"Unsupported tree option: {arg}") + else: + path = arg + i += 1 + scope = self.filesystem.resolve_query_scope(path) + if scope.metadata_axis is not None: + rows, has_more = self.filesystem.scope_metadata_values( + scope, + page=page, + page_size=self.TREE_VALUE_PAGE_SIZE, + ) + root = self._scope_root(scope, file_count=self.filesystem.scope_file_count(scope)) + root["folders"] = [ + { + "path": self._metadata_value_path( + scope.path.rsplit("/", 1)[0] or "/", + scope.metadata_axis, + row["value"], + ), + "name": str(row["value"]), + "type": "metadata_value", + "value": row["value"], + "file_count": row["file_count"], + "folders": [], + } + for row in rows + ] + root["children_count"] = len(root["folders"]) + data = { + "tree": root, + "total_folders": len(root["folders"]), + "depth": depth, + "truncated": has_more, + "pagination": { + "page": page, + "page_size": self.TREE_VALUE_PAGE_SIZE, + "has_more": has_more, + "next_page": page + 1 if has_more else None, + }, + } + next_steps = [ + f'browse {shlex.quote(scope.path)}/ ""' + ] + return data, next_steps + page_size = self.TREE_VALUE_PAGE_SIZE + needed = page * page_size + 1 + folders = self.filesystem.scope_folders( + scope, + max_depth=depth, + limit=max(self.MAX_TREE_FOLDERS, needed), + ) + files = [ + self._tree_file(row) + for row in self.filesystem.scope_files(scope, limit=needed) + ] + axes = self.filesystem.scope_metadata_axes(scope) + tree, has_more = self._folder_tree( + scope, + folders, + files, + axes, + page=page, + page_size=page_size, + ) + data = { + "tree": tree, + "total_folders": len(folders) + len(axes), + "depth": depth, + "truncated": has_more or len(folders) >= self.MAX_TREE_FOLDERS, + } + if has_more or page > 1: + data["pagination"] = { + "page": page, + "page_size": page_size, + "has_more": has_more, + "next_page": page + 1 if has_more else None, + } + next_steps = [f'browse {shlex.quote(scope.path)} ""'] + return data, next_steps + + def _cmd_browse(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: + recursive = False + where = None + page = 1 + positionals: list[str] = [] + i = 0 + while i < len(args): + arg = args[i] + if arg in {"-R", "--recursive"}: + recursive = True + elif arg == "--where": + i, where = self._option_value(args, i, "browse --where") + elif arg == "--page": + i, value = self._option_value(args, i, "browse --page") + page = self._parse_positive_int(value, "browse --page") + elif arg == "--space": + raise PIFSCommandError("browse --space is removed; browse uses summary retrieval only") + elif arg in {"--limit", "--offset", "--query"}: + raise PIFSCommandError(f"browse does not support {arg}; use --page N") + elif arg.startswith("-"): + raise PIFSCommandError(f"Unsupported browse option: {arg}") + else: + positionals.append(arg) + i += 1 + if len(positionals) < 2 or not str(positionals[1]).strip(): + raise PIFSCommandError('browse requires a query: browse ""') + if len(positionals) > 2: + raise PIFSCommandError('browse accepts a folder and one quoted query') + path, query = positionals + if not str(path).startswith("/"): + raise PIFSCommandError("browse target must be a PIFS folder path like /documents") + scope = self.filesystem.resolve_query_scope(path) + if scope.metadata_axis is not None: + raise PIFSCommandError( + "Metadata axis paths require @field/value; run tree /@field to inspect values." + ) + merged_filter = self.filesystem.merge_scope_filter(scope, where) + effective_recursive = recursive or bool(scope.metadata_filter) + payload = self.filesystem.browse_semantic_files( + scope.path, + query, + recursive=effective_recursive, + page=page, + metadata_filter=where, + ) + documents = [self._document_hit(row) for row in payload.get("data", [])] + next_steps = [] + if payload.get("has_more"): + next_steps.append(self._browse_command(path, query, recursive=recursive, where=where, page=page + 1)) + scope_payload = { + "folder": scope.folder_path, + "recursive": effective_recursive, + "query": query, + "where": self._json_filter(where), + "retrieval": "summary", + } + if scope.path != scope.folder_path or merged_filter is not None: + scope_payload.update( + { + "path": scope.path, + "folder_path": scope.folder_path, + "metadata_filter": merged_filter, + } + ) + return { + "documents": documents, + "pagination": { + "page": page, + "page_size": self.BROWSE_PAGE_SIZE, + "has_more": bool(payload.get("has_more")), + "next_page": page + 1 if payload.get("has_more") else None, + }, + "scope": scope_payload, + }, next_steps + + def _cmd_stat(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: + if any(arg == "--schema" for arg in args): + raise PIFSCommandError("stat --schema is removed") + if any(arg == "--field" for arg in args): + raise PIFSCommandError("stat --field is removed") + if any(arg.startswith("-") for arg in args): + raise PIFSCommandError("stat accepts only one document target") + if len(args) != 1: + raise PIFSCommandError("stat accepts exactly one document target") + target = args[0] + return {"document": self._document_stat(target)}, [] + + def _cmd_cat(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: + if not args: + raise PIFSCommandError("cat requires a document target") + target = args[0] + if target.startswith("-"): + raise PIFSCommandError("cat syntax is target-first: cat --structure or cat --page N[-M]") + if "--all" in args or "--range" in args: + raise PIFSCommandError("cat --all and cat --range are removed; use --structure or --page") + if len(args) == 2 and args[1] == "--structure": + payload = self.filesystem.pageindex_structure(target) + return { + "document": self._document_from_structural_payload(payload, target), + "structure": payload.get("structure") if payload.get("available", True) else None, + "pagination": {"available": bool(payload.get("available", True))}, + }, self._page_next_steps(target, payload) + if len(args) == 3 and args[1] == "--page": + pages = args[2] + if not re.fullmatch(r"\d+(?:-\d+)?", pages): + raise PIFSCommandError("cat --page requires one page selector like 31 or 31-33") + start, end = self._parse_numeric_range(pages, "cat --page") + if end - start + 1 > self.MAX_PAGE_SPAN: + raise PIFSCommandError(f"cat --page supports at most {self.MAX_PAGE_SPAN} pages") + payload = self.filesystem.pageindex_pages(target, pages) + return { + "document": self._document_from_structural_payload(payload, target), + "requested_pages": pages, + "returned_pages": payload.get("data", []), + "content": { + "text": payload.get("text", ""), + "available": bool(payload.get("available", True)), + }, + }, [] + raise PIFSCommandError("cat requires either --structure or --page N[-M]") + + def _cmd_grep(self, args: list[str]) -> tuple[dict[str, Any], list[str]]: + if any(arg in {"-R", "-r", "--recursive", "--where"} for arg in args): + raise PIFSCommandError("grep is single-document only: grep ") + if any(arg.startswith("-") for arg in args): + raise PIFSCommandError("grep accepts no options") + if len(args) != 2: + raise PIFSCommandError("grep requires a query and one document target") + query, target = args + if self._is_folder(target): + raise PIFSCommandError("grep requires a resolved file locator, not a folder") + return { + "document": self._document_stat(target), + "matches": self._grep_file_matches(target, query), + }, [] + + def _folder_tree( + self, + scope: Any, + folders: list[dict[str, Any]], + files: list[dict[str, Any]], + axes: list[dict[str, Any]], + *, + page: int, + page_size: int, + ) -> tuple[dict[str, Any], bool]: + root_key = self._normalize_folder_path(scope.folder_path) + root = self._scope_root(scope, file_count=self.filesystem.scope_file_count(scope)) + nodes = { + root_key: root + } + for folder in sorted(folders, key=lambda item: item["path"]): + physical_path = self._normalize_folder_path(folder["path"]) + nodes[physical_path] = { + "path": self._scoped_folder_path(scope, physical_path), + "name": folder.get("name") or physical_path.rsplit("/", 1)[-1], + "type": "folder", + "file_count": folder.get("matched_files", 0) or folder.get("file_count", 0), + "children_count": folder.get("children_count", 0), + "folders": [], + } + for folder_path, node in sorted(nodes.items(), key=lambda item: item[0].count("/")): + if folder_path == root_key: + continue + parent = folder_path.rsplit("/", 1)[0] or "/" + nodes.get(parent, nodes[root_key])["folders"].append(node) + axis_nodes = [ + { + "path": self._join_scope_path( + scope.path, + f"@{self.filesystem.encode_scope_segment(axis['name'])}", + ), + "name": f"@{axis['name']}", + "type": "metadata_axis", + "value_count": axis["value_count"], + "folders": [], + } + for axis in axes + ] + child_items = ( + [("folder", node) for node in nodes[root_key]["folders"]] + + [("file", file) for file in files] + + [("folder", node) for node in axis_nodes] + ) + offset = (page - 1) * page_size + page_items = child_items[offset : offset + page_size] + nodes[root_key]["folders"] = [node for kind, node in page_items if kind == "folder"] + nodes[root_key]["files"] = [node for kind, node in page_items if kind == "file"] + nodes[root_key]["children_count"] = len(child_items) + return nodes[root_key], len(child_items) > offset + page_size + + def _document_hit(self, row: dict[str, Any]) -> dict[str, Any]: + return { + "path": row.get("path"), + "document_id": row.get("external_id"), + "title": row.get("title"), + "status": row.get("pageindex_tree_status"), + "rank": row.get("rank"), + "similarity": row.get("similarity"), + "summary": row.get("summary", ""), + "metadata": row.get("metadata", {}), + "folder_path": row.get("folder_path"), + "folder_paths": row.get("folder_paths", []), + } + + @staticmethod + def _tree_file(row: dict[str, Any]) -> dict[str, Any]: + return { + "path": row["path"], + "document_id": row.get("external_id"), + "title": row["title"], + "status": row["pageindex_tree_status"], + "type": "file", + "metadata": row.get("metadata", {}), + } + + def _document_stat(self, target: str) -> dict[str, Any]: + file_ref = self.filesystem._resolve_target(target) + entry = self.filesystem.store.get_file(file_ref) + folder_paths = [ + folder["path"] + for folder in self.filesystem.store.folder_memberships(file_ref) + ] + folder_path = self.filesystem._preferred_folder_path( + folder_paths, + entry.folder_path, + entry.folder_path, + ) + path = ( + target + if str(target).startswith("/") + else self.filesystem._stable_file_locator( + file_ref, + entry, + folder_path=folder_path, + ) + ) + return { + "path": path, + "document_id": entry.external_id, + "title": entry.title, + "status": entry.pageindex_tree_status, + "content_type": entry.content_type, + "metadata": entry.metadata, + "metadata_status": entry.metadata_status, + "folder_paths": folder_paths, + } + + def _document_from_structural_payload(self, payload: dict[str, Any], target: str) -> dict[str, Any]: + document = self._document_stat(target) + document["available"] = bool(payload.get("available", True)) + if payload.get("message"): + document["message"] = payload.get("message") + return document + + def _page_next_steps(self, target: str, payload: dict[str, Any]) -> list[str]: + if not payload.get("available", True): + return [] + return [f"cat {shlex.quote(target)} --page "] + + def _grep_file_matches(self, target: str, query: str) -> list[dict[str, Any]]: + file_ref = self.filesystem._resolve_target(target) + matches = [] + for line_number, line in enumerate(self.filesystem.store.read_text(file_ref).splitlines(), 1): + if self._line_matches(line, query): + matches.append({"line": line_number, "text": self._compact_text(line, max_chars=220)}) + if len(matches) >= self.MAX_GREP_MATCHES: + break + return matches + + def _is_folder(self, path: str) -> bool: + try: + self.filesystem.folder_info(path) + return True + except KeyError: + return False + + def _browse_command( + self, + path: str, + query: str, + *, + recursive: bool, + where: str | None, + page: int, + ) -> str: + parts = ["browse"] + if recursive: + parts.append("-R") + parts.extend([shlex.quote(self._normalize_folder_path(path)), shlex.quote(query)]) + if where is not None: + parts.extend(["--where", shlex.quote(where)]) + parts.extend(["--page", str(page)]) + return " ".join(parts) + + @staticmethod + def _json_filter(value: str | None) -> Any: + if value is None: + return None + try: + return json.loads(value) + except json.JSONDecodeError: + return value + + @staticmethod + def _line_matches(line: str, query: str) -> bool: + haystack = line.lower() + needle = query.lower().strip() + if needle and needle in haystack: + return True + terms = [term for term in re.findall(r"[A-Za-z0-9_]+", needle) if term] + return bool(terms) and all(term in haystack for term in terms) + + @staticmethod + def _compact_text(text: str, *, max_chars: int) -> str: + collapsed = re.sub(r"\s+", " ", text or "").strip() + if len(collapsed) <= max_chars: + return collapsed + return collapsed[: max_chars - 3].rstrip() + "..." + + @staticmethod + def _parse_numeric_range(value: str, label: str) -> tuple[int, int]: + try: + if "-" in value: + left, right = value.split("-", 1) + start, end = int(left), int(right) + else: + start = end = int(value) + except ValueError as exc: + raise PIFSCommandError(f"{label} requires a numeric range") from exc + if start < 1 or end < start: + raise PIFSCommandError(f"Invalid {label} range: {value}") + return start, end + + @staticmethod + def _option_value(args: list[str], index: int, label: str) -> tuple[int, str]: + value_index = index + 1 + if value_index >= len(args): + raise PIFSCommandError(f"{label} requires a value") + return value_index, args[value_index] + + @staticmethod + def _parse_positive_int(value: str, label: str) -> int: + try: + parsed = int(value) + except ValueError as exc: + raise PIFSCommandError(f"{label} must be an integer") from exc + if parsed < 1: + raise PIFSCommandError(f"{label} must be at least 1") + return parsed + + @staticmethod + def _normalize_folder_path(path: str) -> str: + value = str(path or "/").strip() + if not value or value == "/": + return "/" + return "/" + value.strip("/") + + def _scope_root(self, scope: Any, *, file_count: int) -> dict[str, Any]: + if getattr(scope, "metadata_axis", None): + name = f"@{scope.metadata_axis}" + node_type = "metadata_axis" + elif getattr(scope, "metadata_filter", None): + name = str(next(reversed(scope.metadata_filter.values()))) + node_type = "metadata_value" + else: + info = self.filesystem.folder_info(scope.folder_path) + name = info.get("name") or ("/" if scope.folder_path == "/" else scope.folder_path.rsplit("/", 1)[-1]) + node_type = "folder" + return { + "path": scope.path, + "name": name, + "type": node_type, + "file_count": file_count, + "children_count": 0, + "folders": [], + } + + def _scoped_folder_path(self, scope: Any, folder_path: str) -> str: + path = self._normalize_folder_path(folder_path) + for field, value in getattr(scope, "metadata_filter", {}).items(): + path = self._metadata_value_path(path, field, value) + return path + + def _metadata_value_path(self, base: str, field: object, value: object) -> str: + axis_path = self._join_scope_path( + base, + f"@{self.filesystem.encode_scope_segment(field)}", + ) + return self._join_scope_path( + axis_path, + self.filesystem.encode_scope_segment(value), + ) + + @staticmethod + def _join_scope_path(base: str, segment: str) -> str: + base_path = str(base or "/").rstrip("/") + if not base_path: + base_path = "/" + if base_path == "/": + return f"/{segment}" + return f"{base_path}/{segment}" + + @staticmethod + def _success(data: dict[str, Any], *, next_steps: list[str] | None = None) -> str: + return json.dumps( + {"success": True, "data": data, "next_steps": next_steps or []}, + ensure_ascii=False, + ) + + @staticmethod + def _error(message: str) -> str: + return json.dumps( + { + "success": False, + "error": {"code": "invalid_command", "message": message}, + "next_steps": [], + }, + ensure_ascii=False, + ) + + @classmethod + def _validate_raw_command(cls, command: str) -> None: + if not command.strip(): + raise PIFSCommandError("Empty command") + if any(token in command for token in cls.FORBIDDEN_SUBSTRINGS): + raise PIFSCommandError("Only PageIndex FileSystem commands are allowed") + + @classmethod + def _validate_tokens(cls, tokens: list[str]) -> None: + if any(token in cls.FORBIDDEN_TOKENS for token in tokens): + raise PIFSCommandError("Only PageIndex FileSystem commands are allowed") + + @staticmethod + def _clean_error_message(exc: BaseException) -> str: + message = str(exc) + if isinstance(exc, KeyError) and len(exc.args) == 1: + message = str(exc.args[0]) + return message or exc.__class__.__name__ diff --git a/pageindex/filesystem/core.py b/pageindex/filesystem/core.py new file mode 100644 index 000000000..fb2c48be3 --- /dev/null +++ b/pageindex/filesystem/core.py @@ -0,0 +1,1906 @@ +from __future__ import annotations + +import json +import os +import shutil +import tempfile +from dataclasses import dataclass, field +from pathlib import Path, PurePosixPath +from typing import TYPE_CHECKING, Any, Optional, Union +from urllib.parse import quote, unquote, urlparse + +from ._projection_topology import ( + projection_database_pair, + projection_database_path_present, + projection_database_paths, +) +from .metadata import MetadataQueryEngine +from .store import ( + SQLiteFileSystemStore, + fingerprint, + make_file_ref, + normalize_path, +) +from .types import PIFSQueryScope + +if TYPE_CHECKING: + from ..client import PageIndexClient + from .semantic_projection import _EmbeddingCacheKey + +PROJECTION_INDEX_STATUSES = { + "not_indexed", + "pending_index", + "generated", + "ready", + "failed", +} + +DEFAULT_EMBEDDING_DIMENSIONS = 1024 +PAGEINDEX_DOCUMENT_SUFFIXES = {".pdf", ".md", ".markdown"} +PAGEINDEX_DOCUMENT_CONTENT_TYPES = { + "application/pdf", + "text/markdown", + "text/x-markdown", + "application/markdown", +} +ADD_FILE_CONTENT_TYPES = { + ".pdf": "application/pdf", + ".md": "text/markdown", + ".markdown": "text/markdown", +} + + +@dataclass +class _RegistrationRollbackSnapshot: + preexisting_pageindex_doc_ids: set[str] + artifact_baselines: dict[Path, bytes | None] = field(default_factory=dict) + records: list[dict[str, Any]] = field(default_factory=list) + new_records: list[dict[str, Any]] = field(default_factory=list) + created_folder_paths: list[str] = field(default_factory=list) + new_metadata_fields: set[str] = field(default_factory=set) + new_cache_keys: set[_EmbeddingCacheKey] = field(default_factory=set) + catalog_rows: dict[str, dict[str, Any]] = field(default_factory=dict) + membership_rows: dict[str, list[dict[str, Any]]] = field(default_factory=dict) + metadata_value_rows: dict[str, list[dict[str, Any]]] = field(default_factory=dict) + projection_rows: dict[ + str, + tuple[dict[str, Any], dict[str, Any]], + ] = field(default_factory=dict) + + +def strip_pageindex_text_fields(value: Any) -> Any: + if isinstance(value, list): + return [strip_pageindex_text_fields(item) for item in value] + if isinstance(value, dict): + return { + key: strip_pageindex_text_fields(item) + for key, item in value.items() + if key != "text" + } + return value + + +class PageIndexFileSystem: + def __init__( + self, + workspace: Union[str, Path], + *, + summary_projection_index_dir: Union[str, Path, None] = None, + summary_projection_embedding_model: str = "text-embedding-3-small", + summary_projection_embedding_dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS, + summary_projection_embedding_timeout: float = 60, + summary_projection_embedding_api_key: str | None = None, + summary_projection_embedding_base_url: str | None = None, + ): + self.workspace = Path(workspace).expanduser() + self.summary_projection_index_dir = ( + Path(summary_projection_index_dir).expanduser() + if summary_projection_index_dir is not None + else self.workspace / "artifacts" / "projection_indexes" + ) + summary_path, cache_path = projection_database_paths( + self.summary_projection_index_dir + ) + catalog_path = self.workspace / "filesystem.sqlite" + catalog_present = catalog_path.exists() or catalog_path.is_symlink() + summary_present = projection_database_path_present(summary_path) + cache_present = projection_database_path_present(cache_path) + if catalog_present or summary_present or cache_present: + SQLiteFileSystemStore.validate_existing_database(catalog_path) + database_pair = projection_database_pair(self.summary_projection_index_dir) + if database_pair is not None: + from .semantic_projection import validate_projection_topology + from ._workspace_consistency import validate_workspace_consistency + + validate_projection_topology(self.summary_projection_index_dir) + validate_workspace_consistency( + catalog_path, + database_pair[0], + ) + elif catalog_present: + from ._workspace_consistency import validate_catalog_without_projection + + validate_catalog_without_projection(catalog_path) + self.store = SQLiteFileSystemStore(self.workspace) + self.metadata = MetadataQueryEngine(self.store) + self.summary_projection: Any | None = None + self.summary_projection_embedding_model = summary_projection_embedding_model + self.summary_projection_embedding_dimensions = summary_projection_embedding_dimensions + self.summary_projection_embedding_timeout = summary_projection_embedding_timeout + self.summary_projection_embedding_api_key = summary_projection_embedding_api_key + self.summary_projection_embedding_base_url = summary_projection_embedding_base_url + + def register_file( + self, + *, + storage_uri: str, + folder_path: Optional[str] = None, + metadata: Optional[dict[str, Any]] = None, + external_id: Optional[str] = None, + title: Optional[str] = None, + content: str = "", + content_type: str | None = None, + source_type: Optional[str] = None, + ) -> str: + return self.register_files( + [ + { + "storage_uri": storage_uri, + "folder_path": folder_path, + "metadata": metadata, + "external_id": external_id, + "title": title, + "content": content, + "content_type": content_type, + "source_type": source_type, + } + ] + )[0] + + def add_file( + self, + physical_path: Union[str, Path], + virtual_target: Union[str, Path], + ) -> dict[str, Any]: + source = Path(physical_path).expanduser() + if not source.is_file(): + raise FileNotFoundError(f"Source file not found: {source}") + suffix = source.suffix.lower() + content_type = ADD_FILE_CONTENT_TYPES.get(suffix) + if content_type is None: + supported = ", ".join(sorted(ADD_FILE_CONTENT_TYPES)) + raise ValueError( + f"Unsupported file type: {suffix or ''}; supported: {supported}" + ) + + folder_path, filename, virtual_path = self._resolve_add_target( + virtual_target, + physical_basename=source.name, + physical_suffix=suffix, + ) + if self.store.file_basename_exists_in_folder(folder_path, filename): + raise FileExistsError(f"File already exists at {virtual_path}") + projection = self._ensure_summary_projection() + add_created_folder_paths = self._add_created_folder_paths(folder_path) + file_ref = make_file_ref(virtual_path.strip("/")) + uploads_dir = self.workspace / "artifacts" / "uploads" + final_dir = uploads_dir / file_ref + final_path = final_dir / filename + final_dir_created = False + catalog_inserted = False + records: list[dict[str, Any]] = [] + cache_keys: set[_EmbeddingCacheKey] = set() + preexisting_cache_keys: set[_EmbeddingCacheKey] = set() + preexisting_pageindex_doc_ids = self._pageindex_cache_doc_ids() + + uploads_dir.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(prefix=f".add-{file_ref}-", dir=uploads_dir) as tmp: + temp_path = Path(tmp) / filename + try: + shutil.copy2(source, temp_path) + if final_dir.exists(): + raise FileExistsError( + f"Workspace artifact already exists for {virtual_path}: {final_dir}" + ) + final_dir.mkdir(parents=True) + final_dir_created = True + os.replace(temp_path, final_path) + + record = self._prepare_file_record( + { + "storage_uri": final_path.as_uri(), + "folder_path": folder_path, + "metadata": {}, + "external_id": None, + "title": filename, + "content": self._add_file_content(final_path, content_type), + "content_type": content_type, + } + ) + records = [record] + cache_keys = projection.cache_keys_for_records(records) + preexisting_cache_keys = projection.existing_cache_keys(cache_keys) + self._require_add_pageindex_ready(record) + self._register_custom_metadata_fields(records) + self.store.insert_files(records) + catalog_inserted = True + if self._complete_summary_projection_index(record): + self.store.update_file_metadata_status( + record["file_ref"], + metadata=record["metadata"], + metadata_status=record["metadata_status"], + ) + self._require_summary_projection_ready(record, operation="add") + self._sync_owned_raw_artifact(record) + self._ensure_add_semantic_retrieval_ready() + except Exception: + if catalog_inserted: + self._cleanup_catalog_record(file_ref) + self._cleanup_summary_projection_records(records) + self._cleanup_summary_projection_cache( + cache_keys - preexisting_cache_keys + ) + self._cleanup_failed_register_artifacts(records) + self._cleanup_pageindex_cache(records, preexisting_pageindex_doc_ids) + self._cleanup_created_folders(add_created_folder_paths) + if final_dir_created: + shutil.rmtree(final_dir, ignore_errors=True) + raise + + info = self.store.file_info(file_ref) + info["path"] = virtual_path + return info + + def register_files(self, files: list[dict[str, Any]]) -> list[str]: + files = [ + { + **file, + "metadata": self._validated_register_metadata(file.get("metadata")), + } + for file in files + ] + rollback = _RegistrationRollbackSnapshot( + preexisting_pageindex_doc_ids=self._pageindex_cache_doc_ids() + ) + try: + for file in files: + rollback.records.append( + self._prepare_file_record( + file, + artifact_baselines=rollback.artifact_baselines, + ) + ) + projection = self._ensure_summary_projection() if rollback.records else None + self._capture_existing_registration_rows(rollback, projection) + rollback.new_records = [ + record + for record in rollback.records + if record["file_ref"] not in rollback.catalog_rows + ] + rollback.created_folder_paths = sorted( + { + path + for record in rollback.records + for path in self._add_created_folder_paths(record["folder_path"]) + }, + key=lambda path: (path.count("/"), path), + ) + rollback.new_metadata_fields = { + name + for name in self._custom_metadata_field_names(rollback.records) + if not self.store.metadata_field_exists(name) + } + if projection is not None: + batch_cache_keys = projection.cache_keys_for_records(rollback.records) + rollback.new_cache_keys = ( + batch_cache_keys - projection.existing_cache_keys(batch_cache_keys) + ) + self._register_custom_metadata_fields(rollback.records) + self.store.insert_files(rollback.records) + for record in rollback.records: + try: + if self._complete_summary_projection_index(record): + self.store.update_file_metadata_status( + record["file_ref"], + metadata=record["metadata"], + metadata_status=record["metadata_status"], + ) + self._require_summary_projection_ready( + record, + operation="registration", + ) + self._sync_owned_raw_artifact(record) + except KeyError: + continue + except Exception: + self._cleanup_summary_projection_records(rollback.new_records) + self._restore_existing_registration_projection(rollback) + self._cleanup_summary_projection_cache(rollback.new_cache_keys) + for record in rollback.new_records: + self._cleanup_catalog_record(str(record["file_ref"])) + self._restore_existing_registration_catalog(rollback) + self._cleanup_created_folders(rollback.created_folder_paths) + self._cleanup_new_metadata_fields(rollback.new_metadata_fields) + self._cleanup_pageindex_cache( + rollback.records, + rollback.preexisting_pageindex_doc_ids, + ) + self._restore_registration_artifact_baselines( + rollback.artifact_baselines + ) + raise + return [record["file_ref"] for record in rollback.records] + + def _ensure_summary_projection(self) -> Any: + return self._open_summary_projection(create=True) + + def _ensure_add_semantic_retrieval_ready(self) -> None: + projection = self._open_summary_projection(create=False) + if not projection.available: + raise RuntimeError("pifs add failed to make the Summary Projection available") + + def _summary_embedding_profile(self) -> Any: + from .semantic_projection import SummaryEmbeddingProfile + + return SummaryEmbeddingProfile( + base_url=self.summary_projection_embedding_base_url, + model=self.summary_projection_embedding_model, + dimensions=self.summary_projection_embedding_dimensions, + timeout=self.summary_projection_embedding_timeout, + api_key=self.summary_projection_embedding_api_key, + ) + + def _open_summary_projection(self, *, create: bool) -> Any: + if self.summary_projection is None: + from .semantic_projection import SummaryProjection + + self.summary_projection = SummaryProjection( + self.summary_projection_index_dir, + profile=self._summary_embedding_profile(), + create=create, + ) + return self.summary_projection + + def _add_created_folder_paths(self, folder_path: str) -> list[str]: + paths = self._folder_ancestor_paths(folder_path) + return [path for path in paths if not self.store.folder_exists(path)] + + @staticmethod + def _folder_ancestor_paths(folder_path: str) -> list[str]: + normalized = normalize_path(folder_path) + if normalized == "/": + return [] + segments = [segment for segment in normalized.strip("/").split("/") if segment] + paths: list[str] = [] + for index in range(1, len(segments) + 1): + paths.append("/" + "/".join(segments[:index])) + return paths + + def resolve_query_scope(self, path: str) -> PIFSQueryScope: + normalized = normalize_path(path) + if self._folder_exists(normalized): + return PIFSQueryScope(path=normalized, folder_path=normalized) + + parts = [part for part in normalized.strip("/").split("/") if part] + folder_path = "/" + remainder = parts + for index in range(len(parts), -1, -1): + candidate = "/" + "/".join(parts[:index]) if index else "/" + if self._folder_exists(candidate): + folder_path = candidate + remainder = parts[index:] + break + + if not remainder: + return PIFSQueryScope(path=normalized, folder_path=folder_path) + + metadata_filter: dict[str, str] = {} + index = 0 + while index < len(remainder): + segment = remainder[index] + if not segment.startswith("@"): + if index == 0: + raise KeyError(f"Unknown folder path: {normalized}") + raise ValueError( + "Metadata axes must come after the physical folder prefix; " + "inspect the physical folder first, then append @field/value segments. " + "Use the path returned by tree for values containing '/'." + ) + axis_segment = segment[1:] + if "=" in axis_segment: + raise ValueError( + "Metadata virtual paths use @field/value; run tree /@field " + "and copy the returned path." + ) + field = unquote(axis_segment) + self.metadata.validate_field_name(field) + if not self.store.metadata_field_exists(field): + raise ValueError("Unknown metadata axis; run tree to inspect available @field axes.") + if field in metadata_filter: + raise ValueError( + "A metadata field can appear only once in a scope path; " + "choose one value or use browse --where for advanced predicates." + ) + value_index = index + 1 + if value_index == len(remainder): + return PIFSQueryScope( + path=normalized, + folder_path=folder_path, + metadata_filter=metadata_filter, + metadata_axis=field, + ) + encoded_value = remainder[value_index] + if encoded_value.startswith("@"): + raise ValueError( + "Metadata axis inspection must be the final path segment; " + "choose a value with @field/value before appending another axis." + ) + metadata_filter[field] = unquote(encoded_value) + index += 2 + + return PIFSQueryScope( + path=normalized, + folder_path=folder_path, + metadata_filter=metadata_filter, + ) + + def merge_scope_filter( + self, + scope: PIFSQueryScope, + metadata_filter: dict[str, Any] | str | None, + ) -> dict[str, Any] | None: + parsed = self.metadata.parse_filter(metadata_filter) + if scope.metadata_axis is not None: + raise ValueError( + "Metadata axis paths require @field/value; run tree /@field to inspect values." + ) + if not parsed: + return dict(scope.metadata_filter) or None + overlap = set(scope.metadata_filter).intersection(self.metadata.filter_fields(parsed)) + if overlap: + raise ValueError( + "Do not constrain the same metadata field in both the path and --where; " + "move the predicate into one place." + ) + if not scope.metadata_filter: + return parsed + return {**scope.metadata_filter, **parsed} + + def scope_file_count(self, scope: PIFSQueryScope) -> int: + return self.store.count_files( + scope={"folder_path": scope.folder_path, "recursive": True}, + metadata_filter=scope.metadata_filter or None, + ) + + def scope_folders( + self, + scope: PIFSQueryScope, + *, + max_depth: int | None, + limit: int, + ) -> list[dict[str, Any]]: + return self.store.find_folders( + scope.folder_path, + metadata_filter=scope.metadata_filter or None, + limit=limit, + max_depth=max_depth, + include_self=False, + ) + + def scope_metadata_axes(self, scope: PIFSQueryScope) -> list[dict[str, Any]]: + return self.store.list_metadata_axes( + scope={"folder_path": scope.folder_path, "recursive": True}, + metadata_filter=scope.metadata_filter or None, + exclude_fields=set(scope.metadata_filter), + ) + + def scope_metadata_values( + self, + scope: PIFSQueryScope, + *, + page: int, + page_size: int, + ) -> tuple[list[dict[str, Any]], bool]: + if scope.metadata_axis is None: + return [], False + rows = self.store.list_metadata_values( + scope.metadata_axis, + scope={"folder_path": scope.folder_path, "recursive": True}, + metadata_filter=scope.metadata_filter or None, + limit=page_size + 1, + offset=(page - 1) * page_size, + ) + has_more = len(rows) > page_size + return rows[:page_size], has_more + + def scope_files(self, scope: PIFSQueryScope, *, limit: int) -> list[dict[str, Any]]: + leaf_items = self._scope_file_leaf_items(scope, limit=self.scope_file_count(scope) + 1) + locator_leaf_by_file_ref = self._scope_locator_leaf_by_file_ref(leaf_items) + files = [] + for row, leaf in leaf_items: + locator_leaf = locator_leaf_by_file_ref[row["file_ref"]] + files.append( + { + "path": self._scope_file_locator(scope, locator_leaf), + "locator_leaf": locator_leaf, + "type": "file", + "file_ref": row["file_ref"], + "external_id": row["external_id"], + "title": leaf, + "pageindex_tree_status": row["pageindex_tree_status"], + "metadata": row["metadata"], + } + ) + files = sorted( + files, + key=lambda item: ( + str(item["title"]).lower(), + item["path"], + item["file_ref"], + ), + ) + return files[:limit] + + def scope_file_locator(self, scope: PIFSQueryScope, file_ref: str, leaf: str) -> str: + leaf_items = self._scope_file_leaf_items(scope, limit=self.scope_file_count(scope) + 1) + return self._scope_file_locator( + scope, + self._scope_locator_leaf_by_file_ref(leaf_items).get(file_ref, leaf), + ) + + def _scope_file_leaf_items(self, scope: PIFSQueryScope, *, limit: int) -> list[tuple[dict[str, Any], str]]: + recursive = bool(scope.metadata_filter) + rows = self.store.list_files( + scope={"folder_path": scope.folder_path, "recursive": recursive}, + metadata_filter=scope.metadata_filter or None, + limit=limit, + ) + items = [] + for row in rows: + folder_paths = [ + folder["path"] + for folder in self.store.folder_memberships(row["file_ref"]) + ] + folder_path = self._preferred_folder_path( + folder_paths, + scope.folder_path, + row["folder_path"], + ) + leaf = self.store.membership_display_name(row["file_ref"], folder_path) or row["title"] + items.append((row, leaf)) + return items + + @staticmethod + def _scope_leaf_counts(leaf_items: list[tuple[dict[str, Any], str]]) -> dict[str, int]: + counts: dict[str, int] = {} + for _, leaf in leaf_items: + counts[leaf] = counts.get(leaf, 0) + 1 + return counts + + @classmethod + def _scope_locator_leaf_by_file_ref(cls, leaf_items: list[tuple[dict[str, Any], str]]) -> dict[str, str]: + leaf_counts = cls._scope_leaf_counts(leaf_items) + reserved = {leaf for leaf, count in leaf_counts.items() if count == 1} + used: set[str] = set() + next_suffix: dict[str, int] = {} + locator_leaf_by_file_ref: dict[str, str] = {} + for row, leaf in sorted(leaf_items, key=lambda item: (item[1].lower(), item[1], item[0]["file_ref"])): + if leaf_counts[leaf] == 1: + locator_leaf = leaf + else: + suffix = next_suffix.get(leaf, 1) + locator_leaf = f"{leaf}~{suffix}" + while locator_leaf in used or locator_leaf in reserved: + suffix += 1 + locator_leaf = f"{leaf}~{suffix}" + next_suffix[leaf] = suffix + 1 + used.add(locator_leaf) + locator_leaf_by_file_ref[row["file_ref"]] = locator_leaf + return locator_leaf_by_file_ref + + def scope_stat(self, path: str) -> dict[str, Any]: + scope = self.resolve_query_scope(path) + data = { + "path": scope.path, + "folder_path": scope.folder_path, + "metadata_filter": dict(scope.metadata_filter), + "file_count": self.scope_file_count(scope), + "available_axes": [item["name"] for item in self.scope_metadata_axes(scope)], + } + if scope.metadata_axis is not None: + data["metadata_axis"] = scope.metadata_axis + return data + + @staticmethod + def encode_scope_segment(segment: Any) -> str: + value = str(segment) + if value in {".", ".."}: + return value.replace(".", "%2E") + return quote(value, safe="") + + def browse_semantic_files( + self, + path: str, + query: str, + *, + recursive: bool = False, + page: int = 1, + metadata_filter: Optional[dict[str, Any] | str] = None, + ) -> dict[str, Any]: + page_size = 10 + path = normalize_path(path) + query_scope = self.resolve_query_scope(path) + self.store.folder_info(query_scope.folder_path) + query_text = self._query_text(query).strip() + if not query_text: + raise ValueError("browse requires a query") + if page < 1: + raise ValueError("browse --page must be at least 1") + if self.summary_projection is None: + index_path = self.summary_projection_index_dir / "summary.sqlite" + if not index_path.exists(): + raise ValueError("browse Summary Projection is not available") + projection = self._open_summary_projection(create=False) + if not projection.available: + raise ValueError("browse Summary Projection is not available") + parsed_filter = self.merge_scope_filter(query_scope, metadata_filter) + effective_recursive = recursive or bool(query_scope.metadata_filter) + scope = {"folder_path": query_scope.folder_path, "recursive": effective_recursive} + scope_file_refs = self.store.file_refs_for_scope( + scope=scope, + metadata_filter=parsed_filter, + ) + offset = (page - 1) * page_size + needed = offset + page_size + 1 + candidates = ( + projection.search( + query_text, + limit=needed, + file_refs=scope_file_refs, + ) + if scope_file_refs + else [] + ) + scope_file_ref_set = set(scope_file_refs) + rows: list[dict[str, Any]] = [] + seen: set[str] = set() + for candidate in candidates: + file_ref = candidate.file_ref + if file_ref in seen: + continue + if file_ref not in scope_file_ref_set: + continue + if not self.store.file_matches( + file_ref, + scope=scope, + metadata_filter=parsed_filter, + ): + continue + entry = self.store.get_file(file_ref) + folder_paths = [ + folder["path"] + for folder in self.store.folder_memberships(file_ref) + ] + folder_path = self._preferred_folder_path( + folder_paths, + query_scope.folder_path, + entry.folder_path, + ) + display_title = self.store.membership_display_name(file_ref, folder_path) or entry.title + try: + if query_scope.metadata_filter: + stable_path = self.scope_file_locator(query_scope, file_ref, display_title) + else: + stable_path = self._stable_file_locator( + file_ref, + entry, + folder_path=folder_path, + ) + except RuntimeError: + continue + seen.add(file_ref) + rank = len(rows) + 1 + rows.append( + { + "rank": rank, + "similarity": candidate.similarity, + "path": stable_path, + "file_ref": file_ref, + "external_id": entry.external_id, + "title": display_title, + "pageindex_tree_status": entry.pageindex_tree_status, + "folder_path": folder_path, + "folder_paths": folder_paths, + "summary": str((entry.metadata or {}).get("summary") or ""), + "metadata": entry.metadata, + } + ) + if len(rows) >= needed: + break + page_rows = rows[offset : offset + page_size] + payload = { + "mode": "files", + "retrieval": "summary", + "query": query, + "scope": query_scope.path, + "recursive": effective_recursive, + "page": page, + "page_size": page_size, + "has_more": len(rows) > offset + page_size, + "data": page_rows, + } + if metadata_filter is not None: + payload["where"] = self._metadata_filter_payload(metadata_filter) + return payload + + def folder_info(self, path: str = "/") -> dict[str, Any]: + return self.store.folder_info(path) + + def create_folder( + self, + path: str, + kind: str = "manual", + description: str = "", + metadata: Optional[dict[str, Any]] = None, + ) -> str: + return self.store.create_folder( + path, + kind=kind, + description=description, + metadata=metadata, + ) + + def attach_file_to_folder( + self, + file_ref: str, + folder_path_or_id: str, + metadata: Optional[dict[str, Any]] = None, + ) -> None: + self.store.attach_file_to_folder(file_ref, folder_path_or_id, metadata=metadata) + + def attach_files_to_folders(self, items: list[dict[str, Any]]) -> None: + self.store.attach_files_to_folders(items) + + def set_metadata( + self, + target: str, + metadata: dict[str, Any], + *, + clear: bool = False, + ) -> dict[str, Any]: + if not isinstance(metadata, dict): + raise ValueError("metadata must be a JSON object") + if "summary" in metadata: + raise ValueError("setmeta cannot edit PageIndex summary") + file_ref = self._resolve_target(target) + info = self.store.file_info(file_ref) + replacement = {} if clear else dict(metadata) + for name in replacement: + self.metadata.validate_field_name(str(name)) + existing = dict(info.get("metadata") or {}) + summary = existing.get("summary") + if summary: + replacement["summary"] = summary + self._register_custom_metadata_fields([{"metadata": replacement}]) + self.store.update_file_metadata_status( + file_ref, + metadata=replacement, + metadata_status=dict(info.get("metadata_status") or {}), + ) + updated = self.store.file_info(file_ref) + entry = self.store.get_file(file_ref) + folder_paths = [folder["path"] for folder in updated.get("folders", [])] + folder_path = self._preferred_folder_path( + folder_paths, + entry.folder_path, + entry.folder_path, + ) + updated["path"] = self._stable_file_locator( + file_ref, + entry, + folder_path=folder_path, + ) + return updated + + def pageindex_structure( + self, + target: str, + ) -> dict[str, Any]: + file_ref = self._resolve_target(target) + entry = self.store.get_file(file_ref) + self._require_pageindex_document_file(entry, "cat --structure") + client, doc_id = self._pageindex_client_doc_for_entry(entry) + if doc_id is None: + return self._structural_unavailable( + "structure", + entry, + message=( + "PageIndex structure is not cached for this file in the " + "PageIndexClient workspace." + ), + ) + structure = self._client_json(client.get_document_structure(doc_id)) + if isinstance(structure, dict) and structure.get("error"): + return self._structural_unavailable( + "structure", + entry, + message=str(structure["error"]), + ) + return { + "mode": "structure", + "file_ref": file_ref, + "external_id": entry.external_id, + "status": entry.pageindex_tree_status, + "available": True, + "pageindex_doc_id": doc_id, + "structure": strip_pageindex_text_fields(structure), + } + + def pageindex_pages(self, target: str, pages: str) -> dict[str, Any]: + file_ref = self._resolve_target(target) + entry = self.store.get_file(file_ref) + self._require_pageindex_document_file(entry, "cat --page") + client, doc_id = self._pageindex_client_doc_for_entry(entry) + if doc_id is None: + return self._structural_unavailable( + "page", + entry, + pages=pages, + message=( + "PageIndex page content is not cached for this file in the " + "PageIndexClient workspace." + ), + ) + page_entries = self._client_json(client.get_page_content(doc_id, pages)) + if isinstance(page_entries, dict) and page_entries.get("error"): + return self._structural_unavailable( + "page", + entry, + pages=pages, + message=str(page_entries["error"]), + ) + if not isinstance(page_entries, list) or not page_entries: + return self._structural_unavailable( + "page", + entry, + pages=pages, + message="Requested PageIndex page content is not cached for this file.", + ) + text = "\n\n".join(str(page.get("content") or "") for page in page_entries) + return { + "mode": "page", + "file_ref": file_ref, + "external_id": entry.external_id, + "status": entry.pageindex_tree_status, + "available": True, + "pageindex_doc_id": doc_id, + "pages": pages, + "data": page_entries, + "text": text, + } + + def _require_pageindex_document_file(self, entry: Any, command: str) -> None: + if self._file_format(entry) in {"pdf", "markdown", "pageindex"}: + return + raise ValueError( + f"{command} is only supported for PDF/Markdown PageIndex files; " + f"got title={entry.title!r}, content_type={entry.content_type!r}. " + "Use grep for single-document lexical evidence." + ) + + @classmethod + def _file_format(cls, entry: Any) -> str: + if getattr(entry, "pageindex_doc_id", None) or entry.pageindex_tree_status != "not_built": + return "pageindex" + file_format = cls._content_format(getattr(entry, "title", ""), entry.content_type) + if file_format != "unsupported": + return file_format + return "unsupported" + + @classmethod + def _content_format(cls, filename: Any, content_type: str | None) -> str: + suffix = Path(str(filename or "")).suffix.lower() + normalized_content_type = cls._normalized_content_type(content_type) + if suffix == ".pdf" or normalized_content_type == "application/pdf": + return "pdf" + if ( + suffix in PAGEINDEX_DOCUMENT_SUFFIXES + or normalized_content_type in PAGEINDEX_DOCUMENT_CONTENT_TYPES + ): + return "markdown" + return "unsupported" + + @staticmethod + def _normalized_content_type(content_type: str | None) -> str: + return str(content_type or "").split(";", 1)[0].strip().lower() + + @property + def pageindex_client_workspace(self) -> Path: + return self.workspace / "artifacts" / "pageindex_client" + + def _pageindex_client(self) -> PageIndexClient: + from ..client import PageIndexClient + + workspace = self.pageindex_client_workspace + workspace.mkdir(parents=True, exist_ok=True) + metadata_index = workspace / "_meta.json" + if not metadata_index.exists(): + metadata_index.write_text("{}\n", encoding="utf-8") + return PageIndexClient(workspace=str(workspace)) + + def _pageindex_client_doc_for_entry(self, entry: Any) -> tuple[PageIndexClient, str | None]: + client = self._pageindex_client() + if not entry.pageindex_doc_id: + return client, None + if entry.pageindex_doc_id not in client.documents: + return client, None + return client, entry.pageindex_doc_id + + def _registration_pageindex_pointer( + self, + *, + storage_uri: str, + title: str, + content_type: str, + ) -> tuple[str | None, str, dict[str, Any] | None]: + if self._content_format(title, content_type) not in {"pdf", "markdown"}: + return None, "not_built", None + client = self._pageindex_client() + local_path = self._canonical_storage_uri_path(storage_uri) + cached_doc_id = self._find_cached_pageindex_doc_id(client, local_path) + if cached_doc_id: + return cached_doc_id, "built", None + if local_path is None: + return None, "failed", self._pageindex_tree_failure_record( + source="PageIndexFileSystem.registration", + error_type="UnresolvableStorageUri", + message=( + "storage_uri must resolve to a local file path for " + "PDF/Markdown registration." + ), + ) + try: + doc_id = client.index(local_path) + return doc_id, "built", None + except Exception as exc: + return None, "failed", self._pageindex_tree_failure_record( + source="PageIndexClient.index", + error_type=exc.__class__.__name__, + message=str(exc) or exc.__class__.__name__, + ) + + @staticmethod + def _pageindex_tree_failure_record( + *, + source: str, + error_type: str, + message: str, + ) -> dict[str, Any]: + return { + "status": "failed", + "owner": "pageindex", + "source": source, + "error_type": error_type, + "message": message, + } + + def _find_cached_pageindex_doc_id( + self, + client: PageIndexClient, + local_path: str | None, + ) -> str | None: + if local_path is None: + return None + for doc_id, doc in client.documents.items(): + if self._canonical_path(doc.get("path")) == local_path: + return doc_id + return None + + def _canonical_storage_uri_path(self, storage_uri: str) -> str | None: + parsed = urlparse(storage_uri) + if parsed.scheme == "file": + return self._canonical_path(unquote(parsed.path)) + if storage_uri and not parsed.scheme: + return self._canonical_path(storage_uri) + return None + + @staticmethod + def _title_from_storage_uri(storage_uri: str) -> str: + parsed = urlparse(str(storage_uri or "")) + path = unquote(parsed.path) if parsed.scheme else str(storage_uri or "") + return Path(path).name + + @classmethod + def _infer_content_type(cls, *, title: str, storage_uri: str) -> str: + for filename in (title, cls._title_from_storage_uri(storage_uri)): + suffix = Path(str(filename or "")).suffix.lower() + if suffix == ".pdf": + return "application/pdf" + if suffix in PAGEINDEX_DOCUMENT_SUFFIXES: + return "text/markdown" + return "application/octet-stream" + + @staticmethod + def _canonical_path(path: Any) -> str | None: + if not path: + return None + return str(Path(os.path.expanduser(str(path))).resolve(strict=False)) + + @staticmethod + def _client_json(payload: str) -> Any: + try: + return json.loads(payload) + except json.JSONDecodeError: + return {"error": f"Invalid PageIndexClient JSON response: {payload}"} + + @classmethod + def _resolve_add_target( + cls, + virtual_target: Union[str, Path], + *, + physical_basename: str, + physical_suffix: str, + ) -> tuple[str, str, str]: + raw_target = str(virtual_target).strip() + if not raw_target: + raise ValueError("pifs add target is required") + normalized = normalize_path(raw_target) + posix_target = PurePosixPath(normalized) + raw_looks_like_folder = raw_target.replace("\\", "/").endswith("/") + target_suffix = posix_target.suffix.lower() + if raw_looks_like_folder or target_suffix not in ADD_FILE_CONTENT_TYPES: + folder_path = normalized + filename = physical_basename + else: + if target_suffix != physical_suffix: + raise ValueError( + "pifs add target file extension must match the physical file extension" + ) + folder_path = normalize_path(str(posix_target.parent)) + filename = posix_target.name + cls._validate_add_filename(filename) + virtual_path = cls._join_virtual_file_path(folder_path, filename) + return folder_path, filename, virtual_path + + @staticmethod + def _validate_add_filename(filename: str) -> None: + if not filename or filename in {".", ".."}: + raise ValueError("pifs add target filename is required") + if "/" in filename or "\\" in filename: + raise ValueError("pifs add target filename must be a basename") + + @staticmethod + def _join_virtual_file_path(folder_path: str, filename: str) -> str: + folder_path = normalize_path(folder_path) + if folder_path == "/": + return f"/{filename}" + return f"{folder_path}/{filename}" + + def _add_file_content(self, path: Path, content_type: str) -> str: + if self._content_format(path.name, content_type) == "markdown": + return path.read_text(encoding="utf-8") + return "" + + def _require_add_pageindex_ready(self, record: dict[str, Any]) -> None: + if self._content_format(record["title"], record["content_type"]) not in { + "pdf", + "markdown", + }: + return + if record.get("pageindex_tree_status") == "built" and record.get("pageindex_doc_id"): + return + message = self._pageindex_tree_failure_message(record.get("metadata_status")) or ( + "PageIndex tree was not built" + ) + raise RuntimeError(f"pifs add failed to build PageIndex tree: {message}") + + def _require_summary_projection_ready( + self, + record: dict[str, Any], + *, + operation: str, + ) -> None: + summary_projection = (record.get("metadata_status") or {}).get("summary_projection") + if not summary_projection or not summary_projection.get("requested"): + raise RuntimeError( + f"PIFS {operation} requires a requested summary projection index" + ) + if summary_projection.get("status") != "ready": + detail = summary_projection.get("error") or summary_projection.get("status") + raise RuntimeError( + f"PIFS {operation} failed to build summary projection index: {detail}" + ) + + def _prepare_file_record( + self, + file: dict[str, Any], + *, + artifact_baselines: dict[Path, bytes | None] | None = None, + ) -> dict[str, Any]: + storage_uri = file["storage_uri"] + metadata = self._validated_register_metadata(file.get("metadata")) + external_id = file.get("external_id") + content = file.get("content") or "" + folder_path = normalize_path(file.get("folder_path") or "/") + title = str( + file.get("title") + or metadata.get("title") + or self._title_from_storage_uri(storage_uri) + or external_id + or "" + ).strip() + if not title: + raise ValueError("file title is required") + content_type = file.get("content_type") or self._infer_content_type( + title=title, + storage_uri=storage_uri, + ) + if self._content_format(title, content_type) not in {"pdf", "markdown"}: + raise ValueError("PIFS registration supports PageIndex-backed PDF/Markdown files only") + file_ref = make_file_ref( + str(external_id or self._join_virtual_file_path(folder_path, title).strip("/")) + ) + if artifact_baselines is not None: + self._capture_registration_artifact_baselines( + file_ref, + file, + artifact_baselines, + ) + ( + pageindex_doc_id, + pageindex_tree_status, + pageindex_tree_failure, + ) = self._registration_pageindex_pointer( + storage_uri=storage_uri, + title=title, + content_type=content_type, + ) + if pageindex_tree_status != "built" or not pageindex_doc_id: + message = self._pageindex_tree_failure_message( + {"pageindex_tree": pageindex_tree_failure} + ) or "PageIndex tree was not built" + raise RuntimeError(f"PIFS registration requires PageIndex extraction: {message}") + pageindex_summary = self._pageindex_doc_description(pageindex_doc_id) + if not pageindex_summary: + raise RuntimeError("PIFS registration requires PageIndex doc_description") + metadata["summary"] = pageindex_summary + artifact_content = self._registration_text_artifact_content( + title=title, + content_type=content_type, + pageindex_doc_id=pageindex_doc_id, + pageindex_tree_status=pageindex_tree_status, + fallback_content=content, + ) + source_type = file.get("source_type") + metadata_status = self._metadata_status_state(metadata=metadata) + self._attach_pageindex_tree_failure(metadata_status, pageindex_tree_failure) + indexed_metadata = SQLiteFileSystemStore.indexed_metadata_values(metadata) + text_artifact_path = file.get("text_artifact_path") + owns_text_artifact = text_artifact_path is None + if text_artifact_path is None: + text_artifact_path = self.store.write_text_artifact(file_ref, artifact_content) + raw_artifact_path = file.get("raw_artifact_path") + owns_raw_artifact = False + if raw_artifact_path is None and file.get("write_raw_artifact", True): + raw_artifact_path = self.store.raw_dir / f"{file_ref}.json" + owns_raw_artifact = True + return { + "file_ref": file_ref, + "external_id": external_id, + "storage_uri": storage_uri, + "title": title, + "descriptor": title, + "content_type": content_type, + "source_type": source_type, + "fingerprint": fingerprint(artifact_content), + "text_artifact_path": str(text_artifact_path), + "raw_artifact_path": str(raw_artifact_path) if raw_artifact_path is not None else None, + "pageindex_doc_id": pageindex_doc_id, + "pageindex_tree_status": pageindex_tree_status, + "metadata": metadata, + "metadata_json": json.dumps(metadata, ensure_ascii=False), + "metadata_status": metadata_status, + "metadata_status_json": json.dumps(metadata_status, ensure_ascii=False), + "indexed_metadata": indexed_metadata, + "folder_path": folder_path, + "_pifs_owned_text_artifact": owns_text_artifact, + "_pifs_owned_raw_artifact": owns_raw_artifact, + } + + def _registration_text_artifact_content( + self, + *, + title: str, + content_type: str, + pageindex_doc_id: str | None, + pageindex_tree_status: str, + fallback_content: str, + ) -> str: + if self._content_format(title, content_type) not in {"pdf", "markdown"}: + return fallback_content + if pageindex_tree_status != "built" or not pageindex_doc_id: + return fallback_content + return self._pageindex_extracted_text(pageindex_doc_id) or fallback_content + + def _pageindex_extracted_text(self, doc_id: str) -> str: + client = self._pageindex_client() + if doc_id not in client.documents: + return "" + client._ensure_doc_loaded(doc_id) + doc = client.documents.get(doc_id) or {} + return self._pageindex_pages_text(doc.get("pages")) + + def _pageindex_doc_description(self, doc_id: str) -> str: + client = self._pageindex_client() + if doc_id not in client.documents: + return "" + client._ensure_doc_loaded(doc_id) + doc = client.documents.get(doc_id) or {} + return str(doc.get("doc_description") or "").strip() + + @staticmethod + def _pageindex_pages_text(pages: Any) -> str: + if not isinstance(pages, list): + return "" + parts: list[str] = [] + for page in pages: + if not isinstance(page, dict): + continue + content = str(page.get("content") or "").strip() + if content: + parts.append(content) + return "\n\n".join(parts) + + @staticmethod + def _raw_artifact_payload( + *, + folder_path: str, + metadata: dict[str, Any], + metadata_status: dict[str, Any], + ) -> dict[str, Any]: + return { + "folder_path": folder_path, + "metadata": metadata, + "metadata_status": metadata_status, + } + + def _sync_owned_raw_artifact(self, record: dict[str, Any]) -> None: + raw_artifact_path = record.get("raw_artifact_path") + if self._managed_raw_artifact_path( + str(record["file_ref"]), + raw_artifact_path, + ) is None: + return + record["raw_artifact_path"] = str( + self.store.write_raw_artifact( + record["file_ref"], + self._raw_artifact_payload( + folder_path=record["folder_path"], + metadata=record["metadata"], + metadata_status=record["metadata_status"], + ), + ) + ) + + def _record_from_file_entry(self, entry: Any) -> dict[str, Any]: + metadata_status = self._metadata_status_state(metadata=entry.metadata) + self._attach_pageindex_tree_failure( + metadata_status, + entry.metadata_status.get("pageindex_tree"), + ) + return { + "file_ref": entry.file_ref, + "external_id": entry.external_id, + "storage_uri": entry.storage_uri, + "title": entry.title, + "descriptor": entry.descriptor, + "content_type": entry.content_type, + "source_type": entry.source_type, + "fingerprint": entry.fingerprint, + "text_artifact_path": entry.text_artifact_path, + "raw_artifact_path": entry.raw_artifact_path, + "pageindex_doc_id": entry.pageindex_doc_id, + "pageindex_tree_status": entry.pageindex_tree_status, + "metadata": dict(entry.metadata), + "metadata_json": json.dumps(entry.metadata, ensure_ascii=False), + "metadata_status": metadata_status, + "metadata_status_json": json.dumps(metadata_status, ensure_ascii=False), + "indexed_metadata": SQLiteFileSystemStore.indexed_metadata_values(entry.metadata), + "folder_path": entry.folder_path, + } + + def _complete_summary_projection_index(self, record: dict[str, Any]) -> bool: + metadata_status = record["metadata_status"] + summary_index = metadata_status.get("summary_projection") + if not summary_index or not summary_index.get("requested"): + return False + summary = str(record.get("metadata", {}).get("summary") or "").strip() + if not summary: + return False + if self.summary_projection is None: + raise RuntimeError("PIFS Summary Projection is not open") + try: + result = self.summary_projection.upsert_summary(record) + except Exception as exc: + summary_index["status"] = "failed" + summary_index["error"] = str(exc) + self._refresh_record_metadata_status(record) + raise RuntimeError( + f"PIFS failed to build summary projection index: {exc}" + ) from exc + summary_index.clear() + summary_index.update({"requested": True, **result}) + self._refresh_record_metadata_status(record) + return True + + @staticmethod + def _unlink_artifact(path: Any) -> None: + try: + Path(path).unlink() + except FileNotFoundError: + return + + def _cleanup_failed_register_artifacts(self, records: list[dict[str, Any]]) -> None: + for record in records: + if record.get("_pifs_owned_text_artifact"): + self._unlink_artifact(record["text_artifact_path"]) + if record.get("_pifs_owned_raw_artifact") and record.get("raw_artifact_path"): + self._unlink_artifact(record["raw_artifact_path"]) + + def _capture_registration_artifact_baselines( + self, + file_ref: str, + file: dict[str, Any], + baselines: dict[Path, bytes | None], + ) -> None: + paths = [] + if file.get("text_artifact_path") is None: + paths.append(self.store.text_dir / f"{file_ref}.txt") + raw_artifact_path = file.get("raw_artifact_path") + if raw_artifact_path is None and file.get("write_raw_artifact", True): + raw_artifact_path = self.store.raw_dir / f"{file_ref}.json" + managed_raw_path = self._managed_raw_artifact_path(file_ref, raw_artifact_path) + if managed_raw_path is not None: + paths.append(managed_raw_path) + try: + existing = self.store.get_file(file_ref) + except KeyError: + existing = None + if existing is not None and existing.pageindex_doc_id: + paths.extend( + [ + self.pageindex_client_workspace / "_meta.json", + self.pageindex_client_workspace + / f"{existing.pageindex_doc_id}.json", + ] + ) + for path in paths: + if path not in baselines: + baselines[path] = path.read_bytes() if path.is_file() else None + + def _managed_raw_artifact_path( + self, + file_ref: str, + raw_artifact_path: Any, + ) -> Path | None: + if not raw_artifact_path: + return None + default_path = self.store.raw_dir / f"{file_ref}.json" + if Path(raw_artifact_path).expanduser().resolve(strict=False) != ( + default_path.resolve(strict=False) + ): + return None + return default_path + + def _restore_registration_artifact_baselines( + self, + baselines: dict[Path, bytes | None], + ) -> None: + for path, content in baselines.items(): + if content is None: + self._unlink_artifact(path) + continue + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(content) + + def _cleanup_catalog_record(self, file_ref: str) -> None: + try: + self.store.delete_file(file_ref) + except Exception: + return + + def _capture_existing_registration_rows( + self, + snapshot: _RegistrationRollbackSnapshot, + projection: Any | None, + ) -> None: + file_refs = sorted( + { + str(record.get("file_ref") or "") + for record in snapshot.records + if str(record.get("file_ref") or "") + } + ) + with self.store.connect() as connection: + for file_ref in file_refs: + file_row = connection.execute( + "SELECT * FROM files WHERE file_ref = ?", + (file_ref,), + ).fetchone() + if file_row is None: + continue + snapshot.catalog_rows[file_ref] = dict(file_row) + snapshot.membership_rows[file_ref] = [ + dict(row) + for row in connection.execute( + "SELECT * FROM file_folders WHERE file_ref = ? ORDER BY folder_id", + (file_ref,), + ) + ] + snapshot.metadata_value_rows[file_ref] = [ + dict(row) + for row in connection.execute( + "SELECT * FROM metadata_values WHERE file_ref = ? " + "ORDER BY field_id, value_text, created_at", + (file_ref,), + ) + ] + if not snapshot.catalog_rows: + return + if projection is None: + raise RuntimeError( + "PIFS registration cannot snapshot existing files without a Summary Projection" + ) + with projection.index.connect(read_only=True) as connection: + for file_ref in sorted(snapshot.catalog_rows): + doc = connection.execute( + "SELECT * FROM semantic_index_docs WHERE file_ref = ?", + (file_ref,), + ).fetchone() + vector = ( + None + if doc is None + else connection.execute( + "SELECT rowid, source_type, embedding " + "FROM semantic_index_vec WHERE rowid = ?", + (doc["rowid"],), + ).fetchone() + ) + if doc is None or vector is None: + raise RuntimeError( + "PIFS registration found an incomplete existing Summary Projection; " + "migrate this workspace before retrying." + ) + vector_row = dict(vector) + vector_row["embedding"] = bytes(vector_row["embedding"]) + snapshot.projection_rows[file_ref] = (dict(doc), vector_row) + + def _restore_existing_registration_catalog( + self, + snapshot: _RegistrationRollbackSnapshot, + ) -> None: + if not snapshot.catalog_rows: + return + with self.store.connect() as connection: + for file_ref in sorted(snapshot.catalog_rows): + connection.execute( + "DELETE FROM metadata_values WHERE file_ref = ?", (file_ref,) + ) + connection.execute( + "DELETE FROM file_folders WHERE file_ref = ?", (file_ref,) + ) + connection.execute("DELETE FROM files WHERE file_ref = ?", (file_ref,)) + self._insert_registration_snapshot_row( + connection, + "files", + snapshot.catalog_rows[file_ref], + ) + for row in snapshot.membership_rows[file_ref]: + self._insert_registration_snapshot_row( + connection, + "file_folders", + row, + ) + for row in snapshot.metadata_value_rows[file_ref]: + self._insert_registration_snapshot_row( + connection, + "metadata_values", + row, + ) + + def _restore_existing_registration_projection( + self, + snapshot: _RegistrationRollbackSnapshot, + ) -> None: + if not snapshot.projection_rows: + return + if self.summary_projection is None: + raise RuntimeError( + "PIFS registration cannot restore an unopened Summary Projection" + ) + with self.summary_projection.index.connect() as connection: + for file_ref in sorted(snapshot.projection_rows): + current = connection.execute( + "SELECT rowid FROM semantic_index_docs WHERE file_ref = ?", + (file_ref,), + ).fetchall() + for row in current: + connection.execute( + "DELETE FROM semantic_index_vec WHERE rowid = ?", + (row["rowid"],), + ) + connection.execute( + "DELETE FROM semantic_index_docs WHERE file_ref = ?", + (file_ref,), + ) + doc, vector = snapshot.projection_rows[file_ref] + self._insert_registration_snapshot_row( + connection, + "semantic_index_docs", + doc, + ) + self._insert_registration_snapshot_row( + connection, + "semantic_index_vec", + vector, + ) + + @staticmethod + def _insert_registration_snapshot_row( + connection: Any, + table: str, + row: dict[str, Any], + ) -> None: + columns = list(row) + placeholders = ", ".join("?" for _ in columns) + connection.execute( + f"INSERT INTO {table}({', '.join(columns)}) VALUES ({placeholders})", + [row[column] for column in columns], + ) + + def _cleanup_summary_projection_records( + self, + records: list[dict[str, Any]], + ) -> None: + projection = self.summary_projection + if projection is None: + return + for record in records: + file_ref = str(record.get("file_ref") or "") + if not file_ref: + continue + try: + projection.delete_summary(file_ref) + except Exception: + continue + + def _cleanup_summary_projection_cache( + self, + keys: set[_EmbeddingCacheKey], + ) -> None: + if not keys or self.summary_projection is None: + return + try: + self.summary_projection.delete_cache_keys(keys) + except Exception: + return + + def _cleanup_created_folders(self, folder_paths: list[str]) -> None: + for folder_path in reversed(folder_paths): + try: + self.store.delete_empty_folder(folder_path) + except Exception: + continue + + def _cleanup_new_metadata_fields(self, names: set[str]) -> None: + for name in sorted(names): + try: + self.store.delete_metadata_field_if_unreferenced(name) + except Exception: + continue + + def _pageindex_cache_doc_ids(self) -> set[str]: + workspace = self.pageindex_client_workspace + doc_ids = {path.stem for path in workspace.glob("*.json") if path.name != "_meta.json"} + meta_path = workspace / "_meta.json" + if not meta_path.exists(): + return doc_ids + try: + payload = json.loads(meta_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return doc_ids + if isinstance(payload, dict): + doc_ids.update(str(doc_id) for doc_id in payload) + return doc_ids + + def _cleanup_pageindex_cache( + self, + records: list[dict[str, Any]], + preexisting_doc_ids: set[str], + ) -> None: + doc_ids = sorted(self._pageindex_cache_doc_ids() - preexisting_doc_ids) + for record in records: + doc_id = str(record.get("pageindex_doc_id") or "").strip() + if doc_id and doc_id not in preexisting_doc_ids: + doc_ids.append(doc_id) + doc_ids = sorted(set(doc_ids)) + if not doc_ids: + return + workspace = self.pageindex_client_workspace + for doc_id in doc_ids: + try: + (workspace / f"{doc_id}.json").unlink() + except FileNotFoundError: + pass + except Exception: + continue + meta_path = workspace / "_meta.json" + if not meta_path.exists(): + return + try: + payload = json.loads(meta_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return + if not isinstance(payload, dict): + return + changed = False + for doc_id in doc_ids: + if doc_id in payload: + payload.pop(doc_id, None) + changed = True + if not changed: + return + try: + meta_path.write_text( + json.dumps(payload, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + except OSError: + return + + def _refresh_record_metadata_status( + self, + record: dict[str, Any], + *, + explicit_status: str | None = None, + ) -> None: + metadata_status = record["metadata_status"] + metadata_status["status"] = explicit_status or metadata_status.get("status") or "generated" + self._refresh_summary_projection_status(metadata_status, record["metadata"]) + record["metadata_json"] = json.dumps(record["metadata"], ensure_ascii=False) + record["metadata_status_json"] = json.dumps(metadata_status, ensure_ascii=False) + record["indexed_metadata"] = SQLiteFileSystemStore.indexed_metadata_values(record["metadata"]) + + @classmethod + def _structural_unavailable( + cls, + mode: str, + entry: Any, + *, + message: str, + pages: str | None = None, + ) -> dict[str, Any]: + pageindex_tree_error = cls._pageindex_tree_failure_message(entry.metadata_status) + if pageindex_tree_error and entry.pageindex_tree_status == "failed": + message = f"PageIndex tree build failed: {pageindex_tree_error}" + result = { + "mode": mode, + "file_ref": entry.file_ref, + "external_id": entry.external_id, + "status": entry.pageindex_tree_status, + "available": False, + "message": message, + } + if pageindex_tree_error: + result["pageindex_tree_error"] = pageindex_tree_error + if pages is not None: + result["pages"] = pages + return result + + @staticmethod + def _attach_pageindex_tree_failure( + metadata_status: dict[str, Any], + pageindex_tree_failure: Any, + ) -> None: + if isinstance(pageindex_tree_failure, dict) and pageindex_tree_failure: + metadata_status["pageindex_tree"] = dict(pageindex_tree_failure) + + @staticmethod + def _pageindex_tree_failure_message(metadata_status: Any) -> str | None: + if not isinstance(metadata_status, dict): + return None + pageindex_tree = metadata_status.get("pageindex_tree") + if not isinstance(pageindex_tree, dict): + return None + if pageindex_tree.get("status") != "failed": + return None + message = str(pageindex_tree.get("message") or "").strip() + error_type = str(pageindex_tree.get("error_type") or "").strip() + if error_type and message: + return f"{error_type}: {message}" + return message or error_type or None + + def _resolve_target(self, target: str) -> str: + try: + return self.store.resolve_file_ref(target) + except KeyError: + if not str(target).strip().startswith("/"): + raise + normalized = normalize_path(target) + try: + return self._resolve_scope_file_locator(normalized) + except KeyError: + pass + try: + scope = self.resolve_query_scope(normalized) + except (KeyError, ValueError): + pass + else: + raise ValueError(self._scope_file_required_message(scope.path)) + raise KeyError(f"Unknown file target: {target}") + + def _resolve_scope_file_locator(self, target: str) -> str: + prefix, _, leaf = target.rstrip("/").rpartition("/") + if not leaf: + raise KeyError(f"Unknown file target: {target}") + leaf = unquote(leaf) + scope_path = prefix or "/" + scope = self.resolve_query_scope(scope_path) + if scope.metadata_axis is not None: + raise ValueError(self._scope_file_required_message(target)) + # ponytail: scans the scoped leaf list; add an indexed leaf lookup if huge scoped collisions matter. + matches = [ + item + for item in self.scope_files(scope, limit=self.scope_file_count(scope) + 1) + if item["locator_leaf"] == leaf + ] + if not matches: + raise KeyError(f"Unknown file target: {target}") + if len(matches) > 1: + raise KeyError( + f"Ambiguous file target: {target}. Use tree {scope.path} or " + f'browse {scope.path} "" to copy a disambiguated file locator.' + ) + return matches[0]["file_ref"] + + @staticmethod + def _scope_file_required_message(path: str) -> str: + return ( + f"{path} is a scope, not a file locator; use tree {path} or " + f'browse {path} "" to select a file leaf.' + ) + + @staticmethod + def _metadata_filter_payload(metadata_filter: Any) -> str: + if isinstance(metadata_filter, str): + return metadata_filter + return json.dumps( + metadata_filter, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + + def _stable_file_locator( + self, + file_ref: str, + entry: Any, + *, + folder_path: str | None = None, + ) -> str: + folder_path = normalize_path(folder_path or getattr(entry, "folder_path", None) or "/") + title = str( + self.store.membership_display_name(file_ref, folder_path) + or getattr(entry, "title", "") + or "" + ).strip() + if not title: + raise RuntimeError(f"browse cannot build a virtual path for {file_ref}: missing title") + return self.scope_file_locator( + self.resolve_query_scope(folder_path), + file_ref, + title, + ) + + @staticmethod + def _scope_file_locator(scope: PIFSQueryScope, leaf: Any) -> str: + return PageIndexFileSystem._join_virtual_file_path( + scope.path, + PageIndexFileSystem.encode_scope_segment(str(leaf).strip("/")), + ) + + @staticmethod + def _validated_register_metadata(metadata: Any) -> dict[str, Any]: + if metadata is None: + validated = {} + elif not isinstance(metadata, dict): + raise ValueError("metadata must be a JSON object") + else: + validated = dict(metadata) + try: + json.dumps(validated, ensure_ascii=False) + except (TypeError, ValueError) as exc: + raise ValueError("metadata must be JSON serializable") from exc + if "summary" in validated: + raise ValueError("summary is managed by PageIndex doc_description") + return validated + + def _register_custom_metadata_fields(self, records: list[dict[str, Any]]) -> None: + fields = { + name: {} + for name in self._custom_metadata_field_names(records) + if not self.store.metadata_field_exists(name) + } + if fields: + self.metadata.register_schema({"fields": fields}, source="user") + + def _custom_metadata_field_names(self, records: list[dict[str, Any]]) -> set[str]: + fields = set() + for record in records: + for name in SQLiteFileSystemStore.indexed_metadata_values( + record.get("metadata", {}) + ): + if self.metadata.FIELD_RE.match(str(name)): + fields.add(str(name)) + return fields + + @staticmethod + def _metadata_status_state(*, metadata: dict[str, Any]) -> dict[str, Any]: + state = { + "status": "generated", + "summary_projection": { + "requested": True, + "status": "not_indexed", + "owner": "pifs", + "source": "index", + }, + } + PageIndexFileSystem._refresh_summary_projection_status(state, metadata) + return state + + @staticmethod + def _refresh_summary_projection_status( + metadata_status: dict[str, Any], + metadata: dict[str, Any], + ) -> None: + summary_index = metadata_status.get("summary_projection") + if not summary_index or not summary_index.get("requested"): + return + if "summary" not in metadata: + return + if summary_index.get("status", "not_indexed") == "not_indexed": + summary_index["status"] = "pending_index" + + def _folder_exists(self, path: str) -> bool: + try: + self.store.folder_info(path) + return True + except KeyError: + return False + + @staticmethod + def _query_text(query: Union[str, list[str], None]) -> str: + if query is None: + return "" + if isinstance(query, list): + return " ".join(str(item) for item in query) + return str(query) + + @staticmethod + def _preferred_folder_path( + folder_paths: list[str], + scope_path: Optional[str], + fallback: str, + ) -> str: + if scope_path: + scoped = [ + path + for path in folder_paths + if path == scope_path or path.startswith(f"{scope_path.rstrip('/')}/") + ] + if scoped: + return sorted(scoped, key=lambda item: (len(item), item))[0] + non_root = [path for path in folder_paths if path != "/"] + if non_root: + return sorted(non_root, key=lambda item: (len(item), item))[0] + return fallback diff --git a/pageindex/filesystem/metadata.py b/pageindex/filesystem/metadata.py new file mode 100644 index 000000000..02649915a --- /dev/null +++ b/pageindex/filesystem/metadata.py @@ -0,0 +1,159 @@ +from __future__ import annotations + +import json +import re +from typing import Any + +from .types import MetadataField + + +class MetadataQueryError(ValueError): + pass + + +class MetadataQueryEngine: + FIELD_RE = re.compile(r"^[A-Za-z][A-Za-z0-9_]*$") + OPERATORS = {"$eq", "$ne", "$in", "$contains"} + LOGICAL_OPERATORS = {"$and", "$or"} + FOLDER_SCOPE_FIELD_HINTS = {"path", "folder", "folders", "folder_path", "folder_paths"} + MAX_DEPTH = 5 + + def __init__(self, store: Any): + self.store = store + + def register_schema(self, schema: dict[str, Any], source: str = "manual") -> None: + fields = [] + raw_fields = schema.get("fields", schema) + if not isinstance(raw_fields, dict): + raise MetadataQueryError("metadata schema must contain a fields object") + for name, declaration in raw_fields.items(): + name = str(name) + self.validate_field_name(name) + if isinstance(declaration, str): + description = declaration + elif isinstance(declaration, dict): + description = str(declaration.get("description", "")) + else: + raise MetadataQueryError(f"Invalid schema declaration for field: {name}") + fields.append( + MetadataField( + name=name, + description=description, + source=source, + ) + ) + if fields: + self.store.upsert_metadata_fields(fields) + + def parse_filter(self, value: str | dict[str, Any] | None) -> dict[str, Any] | None: + if value is None or value == "": + return None + if isinstance(value, str): + value = self.parse_dsl(value) + if not isinstance(value, dict): + raise MetadataQueryError("metadata_filter must be a JSON object") + self.validate_filter(value) + return value + + def parse_dsl(self, dsl: str) -> dict[str, Any]: + try: + parsed = json.loads(dsl) + except json.JSONDecodeError as exc: + raise MetadataQueryError( + "metadata DSL must be a JSON object, for example " + '\'{"$and":[{"repo":"redwood"},{"ticker":{"$in":["AAPL","MSFT"]}}]}\'' + ) from exc + if not isinstance(parsed, dict): + raise MetadataQueryError("metadata DSL must be a JSON object") + return parsed + + def validate_filter(self, metadata_filter: dict[str, Any], depth: int = 1) -> None: + if depth > self.MAX_DEPTH: + raise MetadataQueryError(f"metadata_filter nesting depth exceeds {self.MAX_DEPTH}") + if not metadata_filter: + return + for key, condition in metadata_filter.items(): + if key in self.LOGICAL_OPERATORS: + self._validate_logical(key, condition, depth) + continue + self.validate_field(key) + self._validate_field_condition(key, condition) + + def _validate_logical(self, operator: str, condition: Any, depth: int) -> None: + if not isinstance(condition, list) or not condition: + raise MetadataQueryError(f"{operator} requires a non-empty list") + for item in condition: + if not isinstance(item, dict): + raise MetadataQueryError(f"{operator} items must be metadata filter objects") + self.validate_filter(item, depth + 1) + + def _validate_field_condition(self, field: str, condition: Any) -> None: + if not isinstance(condition, dict) or not any( + str(key).startswith("$") for key in condition + ): + self._validate_scalar(condition, context=field) + return + if len(condition) != 1: + raise MetadataQueryError( + f"Field {field} condition must contain exactly one metadata operator" + ) + operator, expected = next(iter(condition.items())) + if operator not in self.OPERATORS: + raise MetadataQueryError(f"Unsupported metadata operator: {operator}") + if operator == "$in": + if not isinstance(expected, list): + raise MetadataQueryError(f"{field} $in requires a list") + for item in expected: + self._validate_scalar(item, context=f"{field} $in") + return + if operator == "$contains": + self._validate_scalar(expected, context=f"{field} $contains") + return + self._validate_scalar(expected, context=f"{field} {operator}") + + def validate_field(self, field: str) -> None: + self.validate_field_name(field) + if not self.store.metadata_field_exists(field): + if field in self.FOLDER_SCOPE_FIELD_HINTS: + raise MetadataQueryError( + f"Unknown metadata field: {field}. Folder paths are positional PIFS paths, " + "not metadata fields; use `tree /documents` to inspect folders, then " + '`browse /documents "" --where JSON` for metadata pruning.' + ) + raise MetadataQueryError(f"Unknown metadata field: {field}") + + def validate_field_name(self, field: str) -> None: + if not self.FIELD_RE.match(field): + raise MetadataQueryError(f"Invalid metadata field: {field}") + + def export_schema(self) -> dict[str, Any]: + fields = {} + for field in self.store.list_metadata_fields(): + fields[field.name] = { + "description": field.description, + "source": field.source, + } + return {"fields": fields} + + def filter_fields(self, metadata_filter: dict[str, Any] | None) -> set[str]: + if not metadata_filter: + return set() + fields = set() + for key, condition in metadata_filter.items(): + if key in self.LOGICAL_OPERATORS: + for item in condition: + if isinstance(item, dict): + fields.update(self.filter_fields(item)) + continue + fields.add(str(key)) + return fields + + @staticmethod + def _validate_scalar(value: Any, *, context: str) -> None: + if isinstance(value, bool): + return + if isinstance(value, (int, float)): + return + if isinstance(value, str): + return + raise MetadataQueryError(f"{context} must be a string, number, or boolean") diff --git a/pageindex/filesystem/semantic_index.py b/pageindex/filesystem/semantic_index.py new file mode 100644 index 000000000..420793e4b --- /dev/null +++ b/pageindex/filesystem/semantic_index.py @@ -0,0 +1,635 @@ +from __future__ import annotations + +import hashlib +import json +import re +import sqlite3 +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import sqlite_vec + +from ._embedding_identity import normalize_base_url, normalize_model +from ._sqlite_schema import sqlite_schema_signature + + +class SemanticIndexError(RuntimeError): + pass + + +SCHEMA_VERSION = 2 +SUMMARY_TABLES = { + "semantic_index_config", + "semantic_index_docs", + "semantic_index_vec", +} +SUMMARY_CONFIG_KEYS = {"adapter", "adapter_version", "dimension", "metadata"} +VEC0_DECLARATION_RE = re.compile( + r""" + ^\s*CREATE\s+VIRTUAL\s+TABLE\s+[\"`\[]?semantic_index_vec[\"`\]]? + \s+USING\s+vec0\s*\(\s* + source_type\s+TEXT\s+PARTITION\s+KEY\s*,\s* + embedding\s+FLOAT\s*\[\s*(\d+)\s*\]\s* + (?:distance_metric\s*=\s*([A-Za-z0-9_-]+)\s*)? + \)\s*$ + """, + re.IGNORECASE | re.VERBOSE, +) + + +@dataclass(frozen=True) +class SemanticIndexRecord: + file_ref: str + vector: list[float] + text: str + external_id: str | None = None + source_type: str = "" + title: str = "" + metadata: dict[str, Any] | None = None + + +@dataclass(frozen=True) +class SemanticSearchResult: + file_ref: str + distance: float + external_id: str | None + source_type: str + title: str + text_hash: str + metadata: dict[str, Any] + + +class SQLiteVecSemanticIndex: + """Rebuildable local semantic index backed by sqlite-vec. + + This is intentionally separate from the PIFS catalog tables. The catalog + remains source of truth; this file is a rebuildable recall index. + """ + + def __init__(self, db_path: str | Path): + self.db_path = Path(db_path).expanduser() + + def reset(self, *, dimension: int, metadata: dict[str, Any] | None = None) -> None: + if dimension <= 0: + raise SemanticIndexError("semantic index dimension must be positive") + self.db_path.parent.mkdir(parents=True, exist_ok=True) + with self.connect() as conn: + self._create_schema(conn, dimension=dimension, metadata=metadata or {}) + + def validate(self, expected_identity: dict[str, Any]) -> dict[str, Any]: + info = self.validate_schema() + if info["metadata"] != expected_identity: + raise SemanticIndexError( + "Incompatible PIFS Summary Embedding Profile; migrate the workspace or " + "use the base URL, model, and dimensions that created this projection." + ) + if int(info["dimension"]) != int(expected_identity["dimensions"]): + raise SemanticIndexError( + "Incompatible PIFS Summary Projection dimensions; migrate the workspace " + "or use the matching embedding profile." + ) + return info + + def validate_schema(self) -> dict[str, Any]: + try: + with self.connect(read_only=True) as conn: + version = int(conn.execute("PRAGMA user_version").fetchone()[0]) + tables = self._logical_tables(conn) + config = self._config(conn) + info = self._info_from_connection(conn, config) + actual = self._schema_signature(conn, tables) + dimension = int(info["dimension"]) + self._validate_projection_rows(conn, dimension=dimension) + metadata = info["metadata"] + if ( + version != SCHEMA_VERSION + or tables != SUMMARY_TABLES + or set(config) != SUMMARY_CONFIG_KEYS + or config["adapter"] != "sqlite-vec" + or not config["adapter_version"] + or dimension <= 0 + or set(metadata) != {"base_url", "model", "dimensions"} + or not isinstance(metadata["base_url"], str) + or not metadata["base_url"].strip() + or normalize_base_url(metadata["base_url"]) != metadata["base_url"] + or not isinstance(metadata["model"], str) + or not metadata["model"].strip() + or normalize_model(metadata["model"]) != metadata["model"] + or isinstance(metadata["dimensions"], bool) + or int(metadata["dimensions"]) != dimension + or actual["vec0"] is None + or actual["vec0"]["dimension"] != dimension + or actual["vec0"]["distance_metric"] != "l2" + ): + raise self._incompatible_schema_error() + with self._memory_connection() as expected_conn: + self._create_schema( + expected_conn, + dimension=dimension, + metadata=metadata, + ) + expected = self._schema_signature( + expected_conn, + self._logical_tables(expected_conn), + ) + if actual != expected: + raise self._incompatible_schema_error() + except SemanticIndexError: + raise + except (json.JSONDecodeError, sqlite3.Error, TypeError, ValueError) as exc: + raise self._incompatible_schema_error() from exc + return info + + def upsert_many(self, records: list[SemanticIndexRecord]) -> int: + if not records: + return 0 + dimension = self.dimension() + with self.connect() as conn: + inserted = 0 + for record in records: + if len(record.vector) != dimension: + raise SemanticIndexError( + f"vector dimension mismatch for {record.file_ref}: " + f"expected {dimension}, got {len(record.vector)}" + ) + rowid = self._upsert_doc(conn, record) + conn.execute("DELETE FROM semantic_index_vec WHERE rowid = ?", (rowid,)) + conn.execute( + "INSERT INTO semantic_index_vec(rowid, source_type, embedding) VALUES (?, ?, ?)", + ( + rowid, + record.source_type, + sqlite_vec.serialize_float32(record.vector), + ), + ) + inserted += 1 + conn.commit() + return inserted + + def delete_file_refs(self, file_refs: list[str]) -> int: + refs = [str(file_ref) for file_ref in file_refs if str(file_ref)] + if not refs: + return 0 + placeholders = ", ".join("?" for _ in refs) + with self.connect() as conn: + rows = conn.execute( + f""" + SELECT rowid + FROM semantic_index_docs + WHERE file_ref IN ({placeholders}) + """, + refs, + ).fetchall() + rowids = [int(row["rowid"]) for row in rows] + if not rowids: + return 0 + rowid_placeholders = ", ".join("?" for _ in rowids) + conn.execute( + f"DELETE FROM semantic_index_vec WHERE rowid IN ({rowid_placeholders})", + rowids, + ) + conn.execute( + f"DELETE FROM semantic_index_docs WHERE rowid IN ({rowid_placeholders})", + rowids, + ) + conn.commit() + return len(rowids) + + def search( + self, + vector: list[float], + *, + limit: int = 10, + filters: dict[str, Any] | None = None, + fetch_multiplier: int = 20, + ) -> list[SemanticSearchResult]: + dimension = self.dimension() + if len(vector) != dimension: + raise SemanticIndexError( + f"query vector dimension mismatch: expected {dimension}, got {len(vector)}" + ) + raw_filters = filters or {} + source_types = _source_type_filters(raw_filters) + file_refs = _file_ref_filters(raw_filters) + if file_refs == []: + return [] + with self.connect() as conn: + if file_refs is not None: + _install_file_ref_filter_table(conn, file_refs) + rows = [] + if source_types: + for source_type in source_types: + fetch_k = self._search_fetch_k( + conn, + limit, + fetch_multiplier, + exact_file_ref_filter=file_refs is not None, + source_type=source_type, + ) + if fetch_k <= 0: + continue + rows.extend( + conn.execute( + f""" + SELECT + d.file_ref, + d.external_id, + d.source_type, + d.title, + d.text_hash, + d.metadata_json, + v.distance + FROM semantic_index_vec v + JOIN semantic_index_docs d ON d.rowid = v.rowid + WHERE v.embedding MATCH ? AND k = ? AND v.source_type = ? + {_file_ref_filter_sql(file_refs)} + ORDER BY v.distance + """, + (sqlite_vec.serialize_float32(vector), fetch_k, source_type), + ).fetchall() + ) + rows.sort(key=lambda row: float(row["distance"])) + else: + fetch_k = self._search_fetch_k( + conn, + limit, + fetch_multiplier, + exact_file_ref_filter=file_refs is not None, + ) + if fetch_k <= 0: + return [] + rows = conn.execute( + f""" + SELECT + d.file_ref, + d.external_id, + d.source_type, + d.title, + d.text_hash, + d.metadata_json, + v.distance + FROM semantic_index_vec v + JOIN semantic_index_docs d ON d.rowid = v.rowid + WHERE v.embedding MATCH ? AND k = ? + {_file_ref_filter_sql(file_refs)} + ORDER BY v.distance + """, + (sqlite_vec.serialize_float32(vector), fetch_k), + ).fetchall() + results: list[SemanticSearchResult] = [] + for row in rows: + metadata = _json_obj(row["metadata_json"]) + if not _matches_filters(row, metadata, filters or {}): + continue + results.append( + SemanticSearchResult( + file_ref=row["file_ref"], + distance=float(row["distance"]), + external_id=row["external_id"], + source_type=row["source_type"], + title=row["title"], + text_hash=row["text_hash"], + metadata=metadata, + ) + ) + if len(results) >= limit: + break + return results + + @staticmethod + def _search_fetch_k( + conn: sqlite3.Connection, + limit: int, + fetch_multiplier: int, + *, + exact_file_ref_filter: bool, + source_type: str | None = None, + ) -> int: + if exact_file_ref_filter: + where = [] + params: list[Any] = [] + if source_type is not None: + where.append("source_type = ?") + params.append(source_type) + where_sql = "WHERE " + " AND ".join(where) if where else "" + return int( + conn.execute( + f"SELECT COUNT(*) FROM semantic_index_docs {where_sql}", + params, + ).fetchone()[0] + ) + return min(4096, max(limit, limit * max(fetch_multiplier, 1))) + + def info(self) -> dict[str, Any]: + with self.connect(read_only=True) as conn: + return self._info_from_connection(conn, self._config(conn)) + + def dimension(self) -> int: + with self.connect() as conn: + row = conn.execute( + "SELECT value FROM semantic_index_config WHERE key = 'dimension'" + ).fetchone() + if row is None: + raise SemanticIndexError( + f"semantic index is not initialized; call reset() first: {self.db_path}" + ) + return int(row["value"]) + + def connect(self, *, read_only: bool = False) -> sqlite3.Connection: + if read_only: + conn = sqlite3.connect( + f"{self.db_path.resolve().as_uri()}?mode=ro", + uri=True, + ) + else: + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + self._load_extension(conn) + return conn + + @staticmethod + def _load_extension(conn: sqlite3.Connection) -> None: + conn.enable_load_extension(True) + sqlite_vec.load(conn) + conn.enable_load_extension(False) + + @classmethod + def _memory_connection(cls) -> sqlite3.Connection: + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + cls._load_extension(conn) + return conn + + @staticmethod + def _create_schema( + conn: sqlite3.Connection, + *, + dimension: int, + metadata: dict[str, Any], + ) -> None: + conn.executescript( + """ + DROP TABLE IF EXISTS semantic_index_vec; + DROP TABLE IF EXISTS semantic_index_docs; + DROP TABLE IF EXISTS semantic_index_config; + CREATE TABLE semantic_index_config ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + CREATE TABLE semantic_index_docs ( + rowid INTEGER PRIMARY KEY, + file_ref TEXT NOT NULL UNIQUE, + external_id TEXT, + source_type TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL DEFAULT '', + text_hash TEXT NOT NULL, + text_chars INTEGER NOT NULL DEFAULT 0, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT DEFAULT CURRENT_TIMESTAMP + ); + CREATE INDEX idx_semantic_index_docs_external_id + ON semantic_index_docs(external_id); + CREATE INDEX idx_semantic_index_docs_source_type + ON semantic_index_docs(source_type); + """ + ) + conn.execute( + "CREATE VIRTUAL TABLE semantic_index_vec USING " + f"vec0(source_type TEXT partition key, embedding float[{dimension}])" + ) + config = { + "dimension": str(dimension), + "adapter": "sqlite-vec", + "adapter_version": sqlite_vec.__version__, + "metadata": json.dumps(metadata, ensure_ascii=False, sort_keys=True), + } + conn.executemany( + "INSERT INTO semantic_index_config(key, value) VALUES (?, ?)", + sorted(config.items()), + ) + conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}") + conn.commit() + + @staticmethod + def _logical_tables(conn: sqlite3.Connection) -> set[str]: + return { + str(row[1]) + for row in conn.execute("PRAGMA table_list") + if row[2] in {"table", "virtual"} + and not str(row[1]).startswith("sqlite_") + and not str(row[1]).startswith("semantic_index_vec_") + } + + @staticmethod + def _config(conn: sqlite3.Connection) -> dict[str, str]: + return { + str(row["key"]): str(row["value"]) + for row in conn.execute( + "SELECT key, value FROM semantic_index_config ORDER BY key" + ) + } + + def _info_from_connection( + self, + conn: sqlite3.Connection, + config: dict[str, str], + ) -> dict[str, Any]: + metadata = json.loads(config.get("metadata", "{}")) + if not isinstance(metadata, dict): + raise self._incompatible_schema_error() + return { + "db_path": str(self.db_path), + "adapter": config.get("adapter", ""), + "adapter_version": config.get("adapter_version", ""), + "dimension": int(config.get("dimension", "0") or 0), + "document_count": int( + conn.execute("SELECT COUNT(*) FROM semantic_index_docs").fetchone()[0] + ), + "metadata": metadata, + } + + @staticmethod + def _schema_signature( + conn: sqlite3.Connection, + tables: set[str], + ) -> dict[str, Any]: + signature = sqlite_schema_signature(conn, tables) + vec_row = conn.execute( + "SELECT sql FROM sqlite_master WHERE type = 'table' " + "AND name = 'semantic_index_vec'" + ).fetchone() + match = VEC0_DECLARATION_RE.fullmatch(str(vec_row[0] if vec_row else "")) + signature["vec0"] = ( + { + "partition_key": ("source_type", "text"), + "embedding_type": "float", + "dimension": int(match.group(1)), + "distance_metric": str(match.group(2) or "l2").lower(), + } + if match + else None + ) + return signature + + @classmethod + def _validate_projection_rows( + cls, + conn: sqlite3.Connection, + *, + dimension: int, + ) -> None: + docs = { + int(row["rowid"]): str(row["source_type"]) + for row in conn.execute( + "SELECT rowid, source_type FROM semantic_index_docs" + ) + } + vectors: dict[int, str] = {} + expected_bytes = dimension * 4 + for row in conn.execute( + "SELECT rowid, source_type, embedding FROM semantic_index_vec" + ): + rowid = int(row["rowid"]) + if len(bytes(row["embedding"])) != expected_bytes: + raise cls._incompatible_schema_error() + vectors[rowid] = str(row["source_type"]) + if docs != vectors: + raise cls._incompatible_schema_error() + + @staticmethod + def _incompatible_schema_error() -> SemanticIndexError: + return SemanticIndexError( + "Incompatible PIFS Summary Projection schema; migrate this workspace " + "with pifs-data/scripts/migrate_pifs_workspace.py before opening it." + ) + + @staticmethod + def text_hash(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + @staticmethod + def _upsert_doc(conn: sqlite3.Connection, record: SemanticIndexRecord) -> int: + existing = conn.execute( + "SELECT rowid FROM semantic_index_docs WHERE file_ref = ?", + (record.file_ref,), + ).fetchone() + metadata_json = json.dumps(record.metadata or {}, ensure_ascii=False, sort_keys=True) + text_hash = SQLiteVecSemanticIndex.text_hash(record.text) + if existing is None: + cursor = conn.execute( + """ + INSERT INTO semantic_index_docs( + file_ref, external_id, source_type, title, + text_hash, text_chars, metadata_json + ) VALUES (?, ?, ?, ?, ?, ?, ?) + """, + ( + record.file_ref, + record.external_id, + record.source_type, + record.title, + text_hash, + len(record.text), + metadata_json, + ), + ) + return int(cursor.lastrowid) + rowid = int(existing["rowid"]) + conn.execute( + """ + UPDATE semantic_index_docs + SET external_id = ?, + source_type = ?, + title = ?, + text_hash = ?, + text_chars = ?, + metadata_json = ?, + updated_at = CURRENT_TIMESTAMP + WHERE rowid = ? + """, + ( + record.external_id, + record.source_type, + record.title, + text_hash, + len(record.text), + metadata_json, + rowid, + ), + ) + return rowid + + +def _json_obj(text: str | None) -> dict[str, Any]: + if not text: + return {} + try: + value = json.loads(text) + except json.JSONDecodeError: + return {} + return value if isinstance(value, dict) else {} + + +def _matches_filters( + row: sqlite3.Row, + metadata: dict[str, Any], + filters: dict[str, Any], +) -> bool: + for key, expected in filters.items(): + actual_key = "file_ref" if key == "file_refs" else key + actual = row[actual_key] if actual_key in row.keys() else metadata.get(actual_key) + if isinstance(expected, list): + if str(actual) not in {str(item) for item in expected}: + return False + elif str(actual) != str(expected): + return False + return True + + +def _source_type_filters(filters: dict[str, Any]) -> list[str]: + value = filters.get("source_type") + if value is None: + return [] + if isinstance(value, list): + return [str(item) for item in value if str(item)] + return [str(value)] if str(value) else [] + + +def _file_ref_filters(filters: dict[str, Any]) -> list[str] | None: + if "file_ref" in filters: + value = filters.get("file_ref") + elif "file_refs" in filters: + value = filters.get("file_refs") + else: + return None + if isinstance(value, list): + return [str(item) for item in value if str(item)] + return [str(value)] if str(value) else [] + + +def _install_file_ref_filter_table(conn: sqlite3.Connection, file_refs: list[str]) -> None: + conn.execute( + """ + CREATE TEMP TABLE IF NOT EXISTS semantic_index_filter_file_refs ( + file_ref TEXT PRIMARY KEY + ) + """ + ) + conn.execute("DELETE FROM semantic_index_filter_file_refs") + conn.executemany( + "INSERT OR IGNORE INTO semantic_index_filter_file_refs(file_ref) VALUES (?)", + [(file_ref,) for file_ref in file_refs], + ) + + +def _file_ref_filter_sql(file_refs: list[str] | None) -> str: + if file_refs is None: + return "" + return ( + "AND EXISTS (" + "SELECT 1 FROM semantic_index_filter_file_refs scope_refs " + "WHERE scope_refs.file_ref = d.file_ref" + ")" + ) diff --git a/pageindex/filesystem/semantic_projection.py b/pageindex/filesystem/semantic_projection.py new file mode 100644 index 000000000..4347469f6 --- /dev/null +++ b/pageindex/filesystem/semantic_projection.py @@ -0,0 +1,520 @@ +from __future__ import annotations + +import sqlite3 +import struct +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from ._embedding_identity import ( + DEFAULT_OPENAI_BASE_URL, + normalize_base_url, + normalize_model, +) +from ._projection_topology import projection_database_pair, projection_database_paths +from ._sqlite_schema import ( + normalized_table_sql, + regular_table_names, + sqlite_schema_signature, +) +from .core import DEFAULT_EMBEDDING_DIMENSIONS +from .semantic_index import ( + SCHEMA_VERSION, + SQLiteVecSemanticIndex, + SemanticIndexRecord, +) + + +SUMMARY_INDEX_NAME = "summary" +_EmbeddingCacheKey = tuple[str, str, int, str] + + +@dataclass(frozen=True) +class SummaryEmbeddingProfile: + base_url: str | None = DEFAULT_OPENAI_BASE_URL + model: str = "text-embedding-3-small" + dimensions: int = DEFAULT_EMBEDDING_DIMENSIONS + timeout: float = 60 + api_key: str | None = None + + def __post_init__(self) -> None: + model = normalize_model(self.model) + if int(self.dimensions) <= 0: + raise ValueError("embedding dimensions must be positive") + if float(self.timeout) <= 0: + raise ValueError("embedding timeout must be positive") + object.__setattr__(self, "base_url", normalize_base_url(self.base_url)) + object.__setattr__(self, "model", model) + object.__setattr__(self, "dimensions", int(self.dimensions)) + object.__setattr__(self, "timeout", float(self.timeout)) + + @property + def identity(self) -> dict[str, Any]: + return { + "base_url": self.base_url, + "model": self.model, + "dimensions": self.dimensions, + } + + +@dataclass(frozen=True) +class SummaryProjectionCandidate: + file_ref: str + distance: float + source_type: str + title: str + metadata: dict[str, Any] + + @property + def similarity(self) -> float: + return 1.0 / (1.0 + max(0.0, self.distance)) + + +class SummaryProjection: + """Own the complete PIFS Summary Projection lifecycle.""" + + def __init__( + self, + index_dir: str | Path, + *, + profile: SummaryEmbeddingProfile, + embedder: Any | None = None, + create: bool = False, + fetch_multiplier: int = 100, + ) -> None: + self.index_dir = Path(index_dir).expanduser() + self.profile = profile + self._embedder_instance = embedder + self.fetch_multiplier = fetch_multiplier + self.index = SQLiteVecSemanticIndex( + self.index_dir / f"{SUMMARY_INDEX_NAME}.sqlite" + ) + cache_path = self.index_dir / "embedding_cache.sqlite" + database_pair = projection_database_pair(self.index_dir) + if database_pair is not None: + self.index.validate(self.profile.identity) + self.embedding_cache = EmbeddingCache(cache_path, create=False) + elif create: + self.index_dir.mkdir(parents=True, exist_ok=True) + try: + self.index.reset( + dimension=self.profile.dimensions, + metadata=self.profile.identity, + ) + self.embedding_cache = EmbeddingCache(cache_path, create=True) + except Exception: + self._cleanup_failed_create() + raise + else: + raise RuntimeError("PIFS Summary Projection is not available") + + def _cleanup_failed_create(self) -> None: + for path in projection_database_paths(self.index_dir): + for suffix in ("", "-journal", "-shm", "-wal"): + try: + Path(f"{path}{suffix}").unlink() + except FileNotFoundError: + continue + + def upsert_summary(self, record: dict[str, Any]) -> dict[str, Any]: + summary = str((record.get("metadata") or {}).get("summary") or "").strip() + if not summary: + return {"status": "skipped", "reason": "missing_summary"} + vector = self.embedding_cache.embed_texts( + [summary], + profile=self.profile, + embedder=self._embedder(), + batch_size=1, + )[0] + count = self.index.upsert_many( + [ + SemanticIndexRecord( + file_ref=str(record["file_ref"]), + vector=vector, + text=summary, + external_id=record.get("external_id"), + source_type=str(record.get("source_type") or ""), + title=str(record.get("title") or ""), + metadata=dict(record.get("metadata") or {}), + ) + ] + ) + return { + "status": "ready", + "indexed_rows": count, + "index_path": str(self.index.db_path), + **self.profile.identity, + } + + def delete_summary(self, file_ref: str) -> int: + return self.index.delete_file_refs([file_ref]) + + def cache_keys_for_records( + self, + records: list[dict[str, Any]], + ) -> set[_EmbeddingCacheKey]: + summaries = [ + str((record.get("metadata") or {}).get("summary") or "").strip() + for record in records + ] + return self.embedding_cache.keys_for_texts( + [summary for summary in summaries if summary], + profile=self.profile, + ) + + def existing_cache_keys( + self, + keys: set[_EmbeddingCacheKey], + ) -> set[_EmbeddingCacheKey]: + return self.embedding_cache.existing_keys(keys) + + def delete_cache_keys(self, keys: set[_EmbeddingCacheKey]) -> int: + return self.embedding_cache.delete_keys(keys) + + def search( + self, + query: str, + *, + limit: int = 10, + file_refs: list[str] | None = None, + ) -> list[SummaryProjectionCandidate]: + query = normalize_text(query) + if not query or not self.available: + return [] + vector = self.embedding_cache.embed_texts( + [query], + profile=self.profile, + embedder=self._embedder(), + batch_size=1, + )[0] + filters = {"file_ref": file_refs} if file_refs is not None else None + return [ + SummaryProjectionCandidate( + file_ref=result.file_ref, + distance=result.distance, + source_type=result.source_type, + title=result.title, + metadata=result.metadata, + ) + for result in self.index.search( + vector, + limit=limit, + filters=filters, + fetch_multiplier=self.fetch_multiplier, + ) + ] + + @property + def available(self) -> bool: + return int(self.index.info().get("document_count") or 0) > 0 + + def info(self) -> dict[str, Any]: + return { + **self.index.info(), + "embedding_identity": self.profile.identity, + "available": self.available, + } + + def _embedder(self) -> Any: + if self._embedder_instance is None: + self._embedder_instance = EmbeddingClient(self.profile) + return self._embedder_instance + + +class EmbeddingCache: + def __init__(self, db_path: Path, *, create: bool) -> None: + self.db_path = db_path + exists = self.db_path.exists() + if exists: + self._validate() + elif create: + self.db_path.parent.mkdir(parents=True, exist_ok=True) + with self.connect() as connection: + self._create_schema(connection) + else: + raise RuntimeError( + "PIFS embedding cache is missing; migrate this workspace with " + "pifs-data/scripts/migrate_pifs_workspace.py before opening it." + ) + + def connect(self, *, read_only: bool = False) -> sqlite3.Connection: + if read_only: + connection = sqlite3.connect( + f"{self.db_path.resolve().as_uri()}?mode=ro", + uri=True, + ) + else: + connection = sqlite3.connect(self.db_path) + connection.row_factory = sqlite3.Row + return connection + + def embed_texts( + self, + texts: list[str], + *, + profile: SummaryEmbeddingProfile, + embedder: Any, + batch_size: int, + ) -> list[list[float]]: + hashes = [SQLiteVecSemanticIndex.text_hash(text) for text in texts] + cached: dict[str, list[float]] = {} + with self.connect() as connection: + for text_hash in sorted(set(hashes)): + row = connection.execute( + """ + SELECT vector_blob + FROM embedding_cache + WHERE base_url = ? AND model = ? AND dimensions = ? AND text_hash = ? + """, + ( + profile.base_url, + profile.model, + profile.dimensions, + text_hash, + ), + ).fetchone() + if row is not None: + cached[text_hash] = decode_vector( + bytes(row["vector_blob"]), profile.dimensions + ) + missing_positions = [ + index for index, text_hash in enumerate(hashes) if text_hash not in cached + ] + for start in range(0, len(missing_positions), max(1, batch_size)): + positions = missing_positions[start : start + max(1, batch_size)] + batch_texts = [texts[index] for index in positions] + vectors = embed_with_retry(embedder, batch_texts) + if len(vectors) != len(positions): + raise ValueError( + "embedding response length mismatch: " + f"requested {len(positions)}, received {len(vectors)}" + ) + for vector in vectors: + if len(vector) != profile.dimensions: + raise ValueError( + "embedding dimension mismatch: " + f"expected {profile.dimensions}, received {len(vector)}" + ) + with self.connect() as connection: + connection.executemany( + """ + INSERT OR REPLACE INTO embedding_cache( + base_url, model, dimensions, text_hash, vector_blob + ) VALUES (?, ?, ?, ?, ?) + """, + [ + ( + profile.base_url, + profile.model, + profile.dimensions, + hashes[index], + encode_vector(vector), + ) + for index, vector in zip(positions, vectors) + ], + ) + for index, vector in zip(positions, vectors): + cached[hashes[index]] = vector + return [cached[text_hash] for text_hash in hashes] + + @staticmethod + def keys_for_texts( + texts: list[str], + *, + profile: SummaryEmbeddingProfile, + ) -> set[_EmbeddingCacheKey]: + return { + ( + str(profile.base_url), + profile.model, + profile.dimensions, + SQLiteVecSemanticIndex.text_hash(text), + ) + for text in texts + } + + def existing_keys( + self, + keys: set[_EmbeddingCacheKey], + ) -> set[_EmbeddingCacheKey]: + existing: set[_EmbeddingCacheKey] = set() + with self.connect(read_only=True) as connection: + for key in sorted(keys): + if connection.execute( + """ + SELECT 1 + FROM embedding_cache + WHERE base_url = ? AND model = ? AND dimensions = ? AND text_hash = ? + """, + key, + ).fetchone() is not None: + existing.add(key) + return existing + + def delete_keys(self, keys: set[_EmbeddingCacheKey]) -> int: + if not keys: + return 0 + with self.connect() as connection: + before = connection.total_changes + connection.executemany( + """ + DELETE FROM embedding_cache + WHERE base_url = ? AND model = ? AND dimensions = ? AND text_hash = ? + """, + sorted(keys), + ) + return connection.total_changes - before + + def _validate(self) -> None: + try: + with self.connect(read_only=True) as connection: + version = int(connection.execute("PRAGMA user_version").fetchone()[0]) + actual = self._schema_signature(connection) + invalid_rows = int( + connection.execute( + "SELECT COUNT(*) FROM embedding_cache " + "WHERE dimensions <= 0 OR length(vector_blob) != dimensions * 4 " + "OR trim(base_url) = '' OR trim(model) = '' OR trim(text_hash) = ''" + ).fetchone()[0] + ) + identities = connection.execute( + "SELECT DISTINCT base_url, model FROM embedding_cache" + ).fetchall() + with sqlite3.connect(":memory:") as expected_connection: + expected_connection.row_factory = sqlite3.Row + self._create_schema(expected_connection) + expected = self._schema_signature(expected_connection) + invalid_identity = any( + normalize_base_url(row["base_url"]) != row["base_url"] + or normalize_model(row["model"]) != row["model"] + for row in identities + ) + except (sqlite3.Error, ValueError) as exc: + raise self._incompatible_schema_error() from exc + if ( + version != SCHEMA_VERSION + or actual != expected + or invalid_rows + or invalid_identity + ): + raise self._incompatible_schema_error() + + @staticmethod + def _create_schema(connection: sqlite3.Connection) -> None: + connection.executescript( + f""" + CREATE TABLE embedding_cache ( + base_url TEXT NOT NULL, + model TEXT NOT NULL, + dimensions INTEGER NOT NULL CHECK(dimensions > 0), + text_hash TEXT NOT NULL, + vector_blob BLOB NOT NULL, + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY(base_url, model, dimensions, text_hash) + ); + PRAGMA user_version = {SCHEMA_VERSION}; + """ + ) + + @staticmethod + def _schema_signature(connection: sqlite3.Connection) -> dict[str, Any]: + tables = regular_table_names(connection) + if tables != {"embedding_cache"}: + return {"tables": tables} + signature = sqlite_schema_signature(connection, tables) + signature["sql"] = normalized_table_sql(connection, "embedding_cache") + return signature + + @staticmethod + def _incompatible_schema_error() -> RuntimeError: + return RuntimeError( + "Incompatible PIFS embedding cache schema; migrate this workspace with " + "pifs-data/scripts/migrate_pifs_workspace.py before opening it." + ) + + +def validate_projection_topology(index_dir: str | Path) -> bool: + index_dir = Path(index_dir).expanduser() + database_pair = projection_database_pair(index_dir) + if database_pair is None: + return False + summary_path, cache_path = database_pair + SQLiteVecSemanticIndex(summary_path).validate_schema() + EmbeddingCache(cache_path, create=False) + return True + + +class EmbeddingClient: + def __init__(self, profile: SummaryEmbeddingProfile) -> None: + from openai import OpenAI + + if not profile.api_key: + raise ValueError("embedding_api_key is required for PIFS embeddings") + self.profile = profile + self.client = OpenAI( + api_key=profile.api_key, + base_url=profile.base_url, + timeout=profile.timeout, + ) + + def embed(self, texts: list[str]) -> list[list[float]]: + response = self.client.embeddings.create( + model=self.profile.model, + input=texts, + dimensions=self.profile.dimensions, + ) + return [ + list(item.embedding) + for item in sorted(response.data, key=lambda item: item.index) + ] + + +def normalize_text(text: str) -> str: + return " ".join(str(text or "").split()) + + +def embed_with_retry( + embedder: Any, + texts: list[str], + *, + max_attempts: int = 8, +) -> list[list[float]]: + for attempt in range(1, max_attempts + 1): + try: + return embedder.embed(texts) + except Exception as exc: + if attempt >= max_attempts or not is_retryable_embedding_error(exc): + raise + time.sleep(min(120.0, 2.0 ** (attempt - 1))) + raise RuntimeError("unreachable embedding retry state") + + +def is_retryable_embedding_error(exc: Exception) -> bool: + retryable = getattr(exc, "retryable", None) + if isinstance(retryable, bool): + return retryable + status_code = getattr(exc, "status_code", None) + try: + status = int(status_code) + except (TypeError, ValueError): + status = None + if status is not None: + if status in {408, 409, 429} or status >= 500: + return True + if 400 <= status < 500: + return False + name = exc.__class__.__name__.lower() + return any(token in name for token in ("timeout", "connection", "ratelimit")) + + +def encode_vector(vector: list[float]) -> bytes: + return struct.pack(f"<{len(vector)}f", *vector) + + +def decode_vector(blob: bytes, dimensions: int) -> list[float]: + if len(blob) != dimensions * 4: + raise ValueError( + f"cached embedding has {len(blob) // 4} dimensions, expected {dimensions}" + ) + return list(struct.unpack(f"<{dimensions}f", blob)) diff --git a/pageindex/filesystem/store.py b/pageindex/filesystem/store.py new file mode 100644 index 000000000..31450e67d --- /dev/null +++ b/pageindex/filesystem/store.py @@ -0,0 +1,1981 @@ +from __future__ import annotations + +import hashlib +import json +import re +import sqlite3 +from pathlib import Path +from typing import Any, Iterable, Optional + +from ._sqlite_schema import regular_table_names, sqlite_schema_signature +from .types import FileEntry, MetadataField + +SCHEMA_VERSION = 2 +CATALOG_TABLES = { + "files", + "folders", + "file_folders", + "metadata_fields", + "metadata_values", +} + + +class SQLiteFileSystemStore: + def __init__(self, workspace: str | Path): + self.workspace = Path(workspace).expanduser() + self.workspace.mkdir(parents=True, exist_ok=True) + self.db_path = self.workspace / "filesystem.sqlite" + self.text_dir = self.workspace / "artifacts" / "text" + self.raw_dir = self.workspace / "artifacts" / "raw" + self.pageindex_client_dir = self.workspace / "artifacts" / "pageindex_client" + self.initialize_schema() + for path in (self.text_dir, self.raw_dir, self.pageindex_client_dir): + path.mkdir(parents=True, exist_ok=True) + + def connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.db_path) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA foreign_keys = ON") + return conn + + def initialize_schema(self) -> None: + if self.db_path.exists() or self.db_path.is_symlink(): + self.validate_existing_database(self.db_path) + return + with self.connect() as conn: + self._create_current_schema(conn) + self.ensure_folder(conn, "/") + conn.execute(f"PRAGMA user_version = {SCHEMA_VERSION}") + + @classmethod + def validate_existing_database(cls, db_path: str | Path) -> None: + db_path = Path(db_path) + if not db_path.is_file() or db_path.stat().st_size <= 0: + raise cls._incompatible_schema_error() + try: + with cls._readonly_connection(db_path) as conn: + version = int(conn.execute("PRAGMA user_version").fetchone()[0]) + actual = cls._schema_signature(conn) + with sqlite3.connect(":memory:") as expected_conn: + expected_conn.row_factory = sqlite3.Row + cls._create_current_schema(expected_conn) + expected = cls._schema_signature(expected_conn) + except sqlite3.Error as exc: + raise cls._incompatible_schema_error() from exc + if version != SCHEMA_VERSION or actual != expected: + raise cls._incompatible_schema_error() + from ._workspace_consistency import validate_catalog_root + + validate_catalog_root(db_path) + + @staticmethod + def _create_current_schema(conn: sqlite3.Connection) -> None: + conn.executescript( + """ + CREATE TABLE IF NOT EXISTS files ( + file_ref TEXT PRIMARY KEY, + external_id TEXT, + storage_uri TEXT NOT NULL, + title TEXT NOT NULL, + descriptor TEXT NOT NULL, + content_type TEXT NOT NULL, + source_type TEXT, + fingerprint TEXT NOT NULL, + text_artifact_path TEXT NOT NULL, + raw_artifact_path TEXT, + pageindex_doc_id TEXT, + pageindex_tree_status TEXT NOT NULL DEFAULT 'not_built', + metadata_json TEXT NOT NULL DEFAULT '{}', + metadata_status_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT DEFAULT CURRENT_TIMESTAMP, + deleted_at TEXT + ); + + CREATE TABLE IF NOT EXISTS folders ( + folder_id TEXT PRIMARY KEY, + parent_id TEXT, + name TEXT NOT NULL, + path TEXT NOT NULL UNIQUE, + description TEXT NOT NULL DEFAULT '', + kind TEXT NOT NULL DEFAULT 'manual', + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY(parent_id) REFERENCES folders(folder_id) + ); + + CREATE TABLE IF NOT EXISTS file_folders ( + file_ref TEXT NOT NULL, + folder_id TEXT NOT NULL, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (file_ref, folder_id), + FOREIGN KEY(file_ref) REFERENCES files(file_ref) ON DELETE CASCADE, + FOREIGN KEY(folder_id) REFERENCES folders(folder_id) ON DELETE CASCADE + ); + + CREATE TABLE IF NOT EXISTS metadata_fields ( + field_id TEXT PRIMARY KEY, + name TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + source TEXT NOT NULL DEFAULT 'manual', + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT DEFAULT CURRENT_TIMESTAMP, + UNIQUE(name) + ); + + CREATE TABLE IF NOT EXISTS metadata_values ( + file_ref TEXT NOT NULL, + field_id TEXT NOT NULL, + value_text TEXT NOT NULL, + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY(file_ref) REFERENCES files(file_ref) ON DELETE CASCADE, + FOREIGN KEY(field_id) REFERENCES metadata_fields(field_id) ON DELETE CASCADE + ); + + CREATE INDEX IF NOT EXISTS idx_files_external_id ON files(external_id); + CREATE INDEX IF NOT EXISTS idx_files_source_type ON files(source_type); + CREATE INDEX IF NOT EXISTS idx_folders_parent_id ON folders(parent_id); + CREATE INDEX IF NOT EXISTS idx_file_folders_folder ON file_folders(folder_id); + CREATE INDEX IF NOT EXISTS idx_metadata_values_field_text ON metadata_values(field_id, value_text); + """ + ) + + @staticmethod + def _readonly_connection(path: Path) -> sqlite3.Connection: + connection = sqlite3.connect( + f"{path.resolve().as_uri()}?mode=ro", + uri=True, + ) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA foreign_keys = ON") + return connection + + @staticmethod + def _schema_signature(conn: sqlite3.Connection) -> dict[str, Any]: + tables = regular_table_names(conn) + return sqlite_schema_signature(conn, tables) + + @staticmethod + def _incompatible_schema_error() -> RuntimeError: + return RuntimeError( + "Incompatible PIFS catalog schema; migrate this workspace with " + "pifs-data/scripts/migrate_pifs_workspace.py before opening it " + f"(expected exact schema version {SCHEMA_VERSION} with tables " + f"{sorted(CATALOG_TABLES)})." + ) + + @staticmethod + def _json_object(value: Any) -> dict[str, Any]: + try: + parsed = json.loads(value or "{}") if isinstance(value, str) else value + except json.JSONDecodeError: + return {} + return parsed if isinstance(parsed, dict) else {} + + @staticmethod + def _columns(conn: sqlite3.Connection, table: str) -> set[str]: + return {row["name"] for row in conn.execute(f"PRAGMA table_info({table})").fetchall()} + + def insert_file(self, record: dict[str, Any]) -> None: + self.insert_files([record]) + + def insert_files(self, records: list[dict[str, Any]]) -> None: + if not records: + return + with self.connect() as conn: + conn.execute("PRAGMA temp_store = MEMORY") + folder_cache: dict[tuple[str, str], str] = {} + file_rows = [] + membership_rows = [] + file_ref_rows = [] + metadata_rows = [] + pending_folder_titles: dict[tuple[str, str], str] = {} + metadata_field_ids = { + row["name"]: row["field_id"] + for row in conn.execute( + "SELECT name, field_id FROM metadata_fields" + ).fetchall() + } + for record in records: + folder_cache_key = (record["folder_path"], record.get("folder_kind", "manual")) + folder_id = folder_cache.get(folder_cache_key) + if folder_id is None: + folder_id = self.ensure_folder( + conn, + record["folder_path"], + kind=record.get("folder_kind", "manual"), + ) + folder_cache[folder_cache_key] = folder_id + self._ensure_title_available_in_folder( + conn, + folder_id=folder_id, + file_ref=record["file_ref"], + title=record["title"], + ) + title_key = (folder_id, str(record["title"])) + existing_file_ref = pending_folder_titles.get(title_key) + if existing_file_ref and existing_file_ref != record["file_ref"]: + target = self._virtual_file_target(conn, folder_id, str(record["title"])) + raise FileExistsError(f"File already exists at {target}") + pending_folder_titles[title_key] = record["file_ref"] + file_rows.append(self._file_insert_values(record)) + membership_rows.append( + ( + record["file_ref"], + folder_id, + json.dumps(record.get("folder_metadata") or {}, ensure_ascii=False), + ) + ) + file_ref_rows.append((record["file_ref"],)) + metadata_rows.extend( + self._metadata_insert_values( + record["file_ref"], + record.get("indexed_metadata", record["metadata"]), + metadata_field_ids, + ) + ) + conn.executemany(self._file_insert_sql(), file_rows) + conn.executemany( + """ + INSERT OR REPLACE INTO file_folders(file_ref, folder_id, metadata_json) + VALUES (?, ?, ?) + """, + membership_rows, + ) + conn.executemany("DELETE FROM metadata_values WHERE file_ref = ?", file_ref_rows) + if metadata_rows: + conn.executemany( + """ + INSERT INTO metadata_values( + file_ref, field_id, value_text + ) VALUES (?, ?, ?) + """, + metadata_rows, + ) + + @staticmethod + def _file_insert_sql() -> str: + columns = [ + "file_ref", + "external_id", + "storage_uri", + "title", + "descriptor", + "content_type", + "source_type", + "fingerprint", + "text_artifact_path", + "raw_artifact_path", + "pageindex_doc_id", + "pageindex_tree_status", + "metadata_json", + "metadata_status_json", + ] + columns.extend(["deleted_at", "updated_at"]) + placeholders = ", ".join(["?"] * (len(columns) - 2) + ["NULL", "CURRENT_TIMESTAMP"]) + update_assignments = [ + f"{column} = excluded.{column}" + for column in columns + if column not in {"file_ref", "deleted_at", "updated_at"} + ] + update_assignments.append("updated_at = CURRENT_TIMESTAMP") + return f""" + INSERT INTO files ({", ".join(columns)}) + VALUES ({placeholders}) + ON CONFLICT(file_ref) DO UPDATE SET + {", ".join(update_assignments)} + """ + + @staticmethod + def _file_insert_values(record: dict[str, Any]) -> tuple[Any, ...]: + values: list[Any] = [ + record["file_ref"], + record["external_id"], + record["storage_uri"], + record["title"], + record["descriptor"], + record["content_type"], + record["source_type"], + record["fingerprint"], + record["text_artifact_path"], + record["raw_artifact_path"], + record.get("pageindex_doc_id"), + record.get("pageindex_tree_status", "not_built"), + record["metadata_json"], + record.get("metadata_status_json", "{}"), + ] + return tuple(values) + + def _metadata_insert_values( + self, + file_ref: str, + metadata: dict[str, Any], + metadata_field_ids: dict[str, str], + ) -> list[tuple[Any, ...]]: + values = [] + for name, value in metadata.items(): + if not self._valid_field_name(name): + continue + field_id = metadata_field_ids.get(name) + if field_id is None: + continue + for item in self._metadata_value_items(value): + values.append( + ( + file_ref, + field_id, + item, + ) + ) + return values + + def create_folder( + self, + path: str, + *, + kind: str = "manual", + description: str = "", + metadata: dict[str, Any] | None = None, + ) -> str: + with self.connect() as conn: + return self.ensure_folder( + conn, + path, + kind=kind, + description=description, + metadata=metadata, + ) + + def attach_file_to_folder( + self, + file_ref: str, + folder_path_or_id: str, + *, + metadata: dict[str, Any] | None = None, + ) -> None: + with self.connect() as conn: + resolved_file_ref = self._resolve_file_ref(conn, file_ref) + folder_id = self._resolve_or_create_folder(conn, folder_path_or_id) + self._ensure_title_available_in_folder( + conn, + folder_id=folder_id, + file_ref=resolved_file_ref, + title=self._file_title(conn, resolved_file_ref), + ) + conn.execute( + """ + INSERT INTO file_folders(file_ref, folder_id, metadata_json) + VALUES (?, ?, ?) + ON CONFLICT(file_ref, folder_id) DO UPDATE SET + metadata_json = excluded.metadata_json + """, + ( + resolved_file_ref, + folder_id, + json.dumps(metadata or {}, ensure_ascii=False), + ), + ) + + def attach_files_to_folders(self, items: list[dict[str, Any]]) -> None: + with self.connect() as conn: + for item in items: + resolved_file_ref = self._resolve_file_ref(conn, item["file_ref"]) + folder_id = self._resolve_or_create_folder(conn, item["folder"]) + self._ensure_title_available_in_folder( + conn, + folder_id=folder_id, + file_ref=resolved_file_ref, + title=self._file_title(conn, resolved_file_ref), + ) + conn.execute( + """ + INSERT INTO file_folders(file_ref, folder_id, metadata_json) + VALUES (?, ?, ?) + ON CONFLICT(file_ref, folder_id) DO UPDATE SET + metadata_json = excluded.metadata_json + """, + ( + resolved_file_ref, + folder_id, + json.dumps(item.get("metadata") or {}, ensure_ascii=False), + ), + ) + + def membership_display_name(self, file_ref: str, folder_path: str) -> str | None: + folder_path = normalize_path(folder_path) + with self.connect() as conn: + row = conn.execute( + """ + SELECT ff.metadata_json, f.title + FROM file_folders ff + JOIN folders fo ON fo.folder_id = ff.folder_id + JOIN files f ON f.file_ref = ff.file_ref + WHERE ff.file_ref = ? + AND fo.path = ? + AND f.deleted_at IS NULL + LIMIT 1 + """, + (file_ref, folder_path), + ).fetchone() + if row is None: + return None + metadata = self._json_object(row["metadata_json"]) + return str(metadata.get("display_name") or row["title"] or "").strip() or None + + def _ensure_title_available_in_folder( + self, + conn: sqlite3.Connection, + *, + folder_id: str, + file_ref: str, + title: str, + ) -> None: + row = conn.execute( + """ + SELECT f.file_ref, fo.path + FROM files f + JOIN file_folders ff ON ff.file_ref = f.file_ref + JOIN folders fo ON fo.folder_id = ff.folder_id + WHERE f.deleted_at IS NULL + AND ff.folder_id = ? + AND f.title = ? + AND f.file_ref != ? + LIMIT 1 + """, + (folder_id, title, file_ref), + ).fetchone() + if row: + raise FileExistsError( + f"File already exists at {self._virtual_file_target(conn, folder_id, title)}" + ) + + @staticmethod + def _virtual_file_target( + conn: sqlite3.Connection, + folder_id: str, + title: str, + ) -> str: + row = conn.execute( + "SELECT path FROM folders WHERE folder_id = ?", + (folder_id,), + ).fetchone() + folder_path = normalize_path(row["path"] if row else "/") + return f"/{title}" if folder_path == "/" else f"{folder_path}/{title}" + + @staticmethod + def _file_title(conn: sqlite3.Connection, file_ref: str) -> str: + row = conn.execute( + "SELECT title FROM files WHERE file_ref = ? AND deleted_at IS NULL", + (file_ref,), + ).fetchone() + if row is None: + raise KeyError(f"Unknown file target: {file_ref}") + return str(row["title"]) + + def replace_metadata_values( + self, + conn: sqlite3.Connection, + file_ref: str, + metadata: dict[str, Any], + ) -> None: + conn.execute("DELETE FROM metadata_values WHERE file_ref = ?", (file_ref,)) + for name, value in metadata.items(): + if not self._valid_field_name(name): + continue + field_id = self._registered_field_id(conn, name) + if field_id is None: + continue + for item in self._metadata_value_items(value): + conn.execute( + """ + INSERT INTO metadata_values( + file_ref, field_id, value_text + ) VALUES (?, ?, ?) + """, + ( + file_ref, + field_id, + item, + ), + ) + + @staticmethod + def _registered_field_id(conn: sqlite3.Connection, name: str) -> str | None: + row = conn.execute( + """ + SELECT field_id + FROM metadata_fields + WHERE name = ? + """, + (name,), + ).fetchone() + return None if row is None else row["field_id"] + + def upsert_metadata_fields( + self, + fields: Iterable[MetadataField], + *, + conn: sqlite3.Connection | None = None, + ) -> None: + owns_connection = conn is None + if conn is None: + conn = self.connect() + try: + for field in fields: + conn.execute( + """ + INSERT INTO metadata_fields( + field_id, name, description, source, updated_at + ) VALUES (?, ?, ?, ?, CURRENT_TIMESTAMP) + ON CONFLICT(name) DO UPDATE SET + description = excluded.description, + source = excluded.source, + updated_at = CURRENT_TIMESTAMP + """, + ( + self.field_id(field.name), + field.name, + field.description, + field.source, + ), + ) + if owns_connection: + conn.commit() + finally: + if owns_connection: + conn.close() + + def metadata_field_exists(self, name: str) -> bool: + with self.connect() as conn: + row = conn.execute( + "SELECT 1 FROM metadata_fields WHERE name = ?", + (name,), + ).fetchone() + return row is not None + + def delete_metadata_field_if_unreferenced(self, name: str) -> bool: + with self.connect() as conn: + row = conn.execute( + "SELECT field_id FROM metadata_fields WHERE name = ?", + (name,), + ).fetchone() + if row is None: + return False + referenced = conn.execute( + "SELECT 1 FROM metadata_values WHERE field_id = ? LIMIT 1", + (row["field_id"],), + ).fetchone() + if referenced is not None: + return False + cursor = conn.execute( + "DELETE FROM metadata_fields WHERE field_id = ?", + (row["field_id"],), + ) + return cursor.rowcount > 0 + + def list_metadata_fields(self) -> list[MetadataField]: + with self.connect() as conn: + rows = conn.execute( + """ + SELECT name, description, source + FROM metadata_fields + ORDER BY name + """ + ).fetchall() + return [ + MetadataField( + name=row["name"], + description=row["description"], + source=row["source"], + ) + for row in rows + ] + + def list_folder( + self, + path: str = "/", + recursive: bool = False, + limit: int = 100, + max_depth: int | None = None, + ) -> dict[str, Any]: + path = normalize_path(path) + if max_depth is not None and max_depth < 0: + raise ValueError("max_depth must be non-negative") + with self.connect() as conn: + folder = self._folder_by_path(conn, path) + if folder is None: + raise KeyError(f"Unknown folder path: {path}") + if recursive: + folder_depth_clause = "" + folder_depth_params: list[Any] = [] + if max_depth is not None: + if max_depth == 0: + folder_depth_clause = "AND 0" + else: + folder_depth_clause = ( + f"AND ({self._folder_depth_sql('fo.path')} - ?) <= ?" + ) + folder_depth_params = [self._folder_depth(path), max_depth] + folder_rows = conn.execute( + f""" + SELECT + fo.folder_id, + fo.parent_id, + fo.name, + fo.path, + fo.description, + fo.kind, + fo.metadata_json, + fo.created_at, + fo.updated_at, + ( + SELECT COUNT(DISTINCT child_ff.file_ref) + FROM file_folders child_ff + JOIN files child_file + ON child_file.file_ref = child_ff.file_ref + AND child_file.deleted_at IS NULL + WHERE child_ff.folder_id = fo.folder_id + ) AS file_count, + ( + SELECT COUNT(*) + FROM folders child_folder + WHERE child_folder.parent_id = fo.folder_id + ) AS children_count + FROM folders fo + WHERE fo.path != ? AND (fo.path LIKE ? ESCAPE '\\') + {folder_depth_clause} + ORDER BY fo.path + LIMIT ? + """, + (path, self._descendant_like(path), *folder_depth_params, limit), + ).fetchall() + file_rows = self._file_rows_for_scope( + conn, + path, + True, + limit, + max_depth=max_depth, + ) + else: + folder_rows = conn.execute( + """ + SELECT + fo.folder_id, + fo.parent_id, + fo.name, + fo.path, + fo.description, + fo.kind, + fo.metadata_json, + fo.created_at, + fo.updated_at, + ( + SELECT COUNT(DISTINCT child_ff.file_ref) + FROM file_folders child_ff + JOIN files child_file + ON child_file.file_ref = child_ff.file_ref + AND child_file.deleted_at IS NULL + WHERE child_ff.folder_id = fo.folder_id + ) AS file_count, + ( + SELECT COUNT(*) + FROM folders child_folder + WHERE child_folder.parent_id = fo.folder_id + ) AS children_count + FROM folders fo + WHERE fo.parent_id = ? + ORDER BY lower(fo.name), fo.name + LIMIT ? + """, + (folder["folder_id"], limit), + ).fetchall() + file_rows = self._file_rows_for_scope(conn, path, False, limit) + return { + "folders": [self._folder_row_to_dict(row) for row in folder_rows], + "files": [self._file_summary(row) for row in file_rows], + } + + def folder_info(self, path: str = "/") -> dict[str, Any]: + path = normalize_path(path) + with self.connect() as conn: + row = conn.execute( + """ + SELECT + fo.folder_id, + fo.parent_id, + fo.name, + fo.path, + fo.description, + fo.kind, + fo.metadata_json, + fo.created_at, + fo.updated_at, + ( + SELECT COUNT(DISTINCT child_ff.file_ref) + FROM file_folders child_ff + JOIN files child_file + ON child_file.file_ref = child_ff.file_ref + AND child_file.deleted_at IS NULL + WHERE child_ff.folder_id = fo.folder_id + ) AS file_count, + ( + SELECT COUNT(*) + FROM folders child_folder + WHERE child_folder.parent_id = fo.folder_id + ) AS children_count + FROM folders fo + WHERE fo.path = ? + """, + (path,), + ).fetchone() + if row is None: + raise KeyError(f"Unknown folder path: {path}") + return self._folder_row_to_dict(row) + + def find_folders( + self, + path: str = "/", + *, + metadata_filter: Optional[dict[str, Any]] = None, + limit: int = 100, + max_depth: int | None = None, + include_self: bool = False, + ) -> list[dict[str, Any]]: + path = normalize_path(path) + if max_depth is not None and max_depth < 0: + raise ValueError("max_depth must be non-negative") + metadata_sql, metadata_params = self._metadata_filter_sql(metadata_filter) + metadata_clause = f"AND {' AND '.join(metadata_sql)}" if metadata_sql else "" + folder_depth_clause = "" + folder_depth_params: list[Any] = [] + if max_depth is not None: + if max_depth == 0 and not include_self: + folder_depth_clause = "AND 0" + else: + folder_depth_clause = f"AND ({self._folder_depth_sql('fo.path')} - ?) <= ?" + folder_depth_params = [self._folder_depth(path), max_depth] + folder_scope_clause = ( + "(fo.path = ? OR fo.path LIKE ? ESCAPE '\\')" + if include_self + else "fo.path != ? AND fo.path LIKE ? ESCAPE '\\'" + ) + sql = f""" + SELECT * + FROM ( + SELECT + fo.folder_id, + fo.parent_id, + fo.name, + fo.path, + fo.description, + fo.kind, + fo.metadata_json, + fo.created_at, + fo.updated_at, + ( + SELECT COUNT(DISTINCT child_ff.file_ref) + FROM file_folders child_ff + JOIN files child_file + ON child_file.file_ref = child_ff.file_ref + AND child_file.deleted_at IS NULL + WHERE child_ff.folder_id = fo.folder_id + ) AS file_count, + ( + SELECT COUNT(*) + FROM folders child_folder + WHERE child_folder.parent_id = fo.folder_id + ) AS children_count, + ( + SELECT COUNT(DISTINCT f.file_ref) + FROM files f + JOIN file_folders matched_ff + ON matched_ff.file_ref = f.file_ref + JOIN folders matched_folder + ON matched_folder.folder_id = matched_ff.folder_id + WHERE f.deleted_at IS NULL + AND ( + matched_folder.folder_id = fo.folder_id + OR matched_folder.path LIKE {self._descendant_like_sql_expr("fo.path")} ESCAPE '\\' + ) + {metadata_clause} + ) AS matched_files + FROM folders fo + WHERE {folder_scope_clause} + {folder_depth_clause} + ) + WHERE matched_files > 0 + ORDER BY path + LIMIT ? + """ + params = [ + *metadata_params, + path, + self._descendant_like(path), + *folder_depth_params, + limit, + ] + with self.connect() as conn: + folder = self._folder_by_path(conn, path) + if folder is None: + raise KeyError(f"Unknown folder path: {path}") + rows = conn.execute(sql, params).fetchall() + return [self._folder_row_to_dict(row) for row in rows] + + def list_files( + self, + *, + scope: Optional[dict[str, Any]] = None, + metadata_filter: Optional[dict[str, Any]] = None, + limit: int = 100, + ) -> list[dict[str, Any]]: + selects = [ + "f.file_ref", + "f.external_id", + "f.title", + "f.descriptor", + "f.pageindex_tree_status", + "f.metadata_json", + "f.metadata_status_json", + "f.created_at", + """ + ( + SELECT display_folder.folder_id + FROM file_folders display_ff + JOIN folders display_folder + ON display_folder.folder_id = display_ff.folder_id + WHERE display_ff.file_ref = f.file_ref + ORDER BY display_folder.path + LIMIT 1 + ) AS folder_id + """, + """ + ( + SELECT display_folder.path + FROM file_folders display_ff + JOIN folders display_folder + ON display_folder.folder_id = display_ff.folder_id + WHERE display_ff.file_ref = f.file_ref + ORDER BY display_folder.path + LIMIT 1 + ) AS folder_path + """, + ] + where = ["f.deleted_at IS NULL"] + params: list[Any] = [] + scope_sql, scope_params = self._scope_sql(scope) + if scope_sql: + where.append(scope_sql) + params.extend(scope_params) + metadata_sql, metadata_params = self._metadata_filter_sql(metadata_filter) + where.extend(metadata_sql) + params.extend(metadata_params) + sql = f""" + SELECT {", ".join(selects)} + FROM files f + WHERE {" AND ".join(where)} + ORDER BY lower(f.title), f.title, f.file_ref + LIMIT ? + """ + params.append(limit) + with self.connect() as conn: + rows = conn.execute(sql, params).fetchall() + return [self._file_summary(row) for row in rows] + + def file_refs_for_scope( + self, + *, + scope: Optional[dict[str, Any]] = None, + metadata_filter: Optional[dict[str, Any]] = None, + ) -> list[str]: + where = ["f.deleted_at IS NULL"] + params: list[Any] = [] + scope_sql, scope_params = self._scope_sql(scope) + if scope_sql: + where.append(scope_sql) + params.extend(scope_params) + metadata_sql, metadata_params = self._metadata_filter_sql(metadata_filter) + where.extend(metadata_sql) + params.extend(metadata_params) + with self.connect() as conn: + rows = conn.execute( + f""" + SELECT DISTINCT f.file_ref + FROM files f + WHERE {" AND ".join(where)} + ORDER BY f.file_ref + """, + params, + ).fetchall() + return [row["file_ref"] for row in rows] + + def count_files( + self, + *, + scope: Optional[dict[str, Any]] = None, + metadata_filter: Optional[dict[str, Any]] = None, + ) -> int: + where = ["f.deleted_at IS NULL"] + params: list[Any] = [] + scope_sql, scope_params = self._scope_sql(scope) + if scope_sql: + where.append(scope_sql) + params.extend(scope_params) + metadata_sql, metadata_params = self._metadata_filter_sql(metadata_filter) + where.extend(metadata_sql) + params.extend(metadata_params) + with self.connect() as conn: + row = conn.execute( + f""" + SELECT COUNT(DISTINCT f.file_ref) AS file_count + FROM files f + WHERE {" AND ".join(where)} + """, + params, + ).fetchone() + return int(row["file_count"] if row is not None else 0) + + def list_metadata_axes( + self, + *, + scope: Optional[dict[str, Any]] = None, + metadata_filter: Optional[dict[str, Any]] = None, + exclude_fields: set[str] | None = None, + ) -> list[dict[str, Any]]: + where = ["f.deleted_at IS NULL", "mv.value_text != ''"] + params: list[Any] = [] + scope_sql, scope_params = self._scope_sql(scope) + if scope_sql: + where.append(scope_sql) + params.extend(scope_params) + metadata_sql, metadata_params = self._metadata_filter_sql(metadata_filter) + where.extend(metadata_sql) + params.extend(metadata_params) + if exclude_fields: + placeholders = ", ".join("?" for _ in sorted(exclude_fields)) + where.append(f"mf.name NOT IN ({placeholders})") + params.extend(sorted(exclude_fields)) + with self.connect() as conn: + rows = conn.execute( + f""" + SELECT mf.name, COUNT(DISTINCT mv.value_text) AS value_count + FROM metadata_values mv + JOIN metadata_fields mf ON mf.field_id = mv.field_id + JOIN files f ON f.file_ref = mv.file_ref + WHERE {" AND ".join(where)} + GROUP BY mf.name + ORDER BY mf.name + """, + params, + ).fetchall() + return [ + {"name": row["name"], "value_count": int(row["value_count"] or 0)} + for row in rows + ] + + def list_metadata_values( + self, + field: str, + *, + scope: Optional[dict[str, Any]] = None, + metadata_filter: Optional[dict[str, Any]] = None, + limit: int = 51, + offset: int = 0, + ) -> list[dict[str, Any]]: + where = [ + "f.deleted_at IS NULL", + "mf.name = ?", + "mv.value_text != ''", + ] + params: list[Any] = [field] + scope_sql, scope_params = self._scope_sql(scope) + if scope_sql: + where.append(scope_sql) + params.extend(scope_params) + metadata_sql, metadata_params = self._metadata_filter_sql(metadata_filter) + where.extend(metadata_sql) + params.extend(metadata_params) + params.extend([limit, offset]) + with self.connect() as conn: + rows = conn.execute( + f""" + SELECT mv.value_text AS value, COUNT(DISTINCT f.file_ref) AS file_count + FROM metadata_values mv + JOIN metadata_fields mf ON mf.field_id = mv.field_id + JOIN files f ON f.file_ref = mv.file_ref + WHERE {" AND ".join(where)} + GROUP BY mv.value_text + ORDER BY file_count DESC, mv.value_text + LIMIT ? OFFSET ? + """, + params, + ).fetchall() + return [ + {"value": row["value"], "file_count": int(row["file_count"] or 0)} + for row in rows + ] + + def _metadata_filter_sql(self, metadata_filter: Optional[dict[str, Any]]) -> tuple[list[str], list[Any]]: + if not metadata_filter: + return [], [] + clause, params = self._compile_metadata_filter(metadata_filter) + return [clause] if clause else [], params + + def _compile_metadata_filter(self, metadata_filter: dict[str, Any]) -> tuple[str, list[Any]]: + clauses = [] + params: list[Any] = [] + for key, condition in metadata_filter.items(): + if key in {"$and", "$or"}: + child_clauses = [] + child_params: list[Any] = [] + for item in condition: + child_clause, item_params = self._compile_metadata_filter(item) + if child_clause: + child_clauses.append(f"({child_clause})") + child_params.extend(item_params) + if child_clauses: + joiner = " AND " if key == "$and" else " OR " + clauses.append(joiner.join(child_clauses)) + params.extend(child_params) + continue + field_clause, field_params = self._compile_metadata_field_filter(key, condition) + clauses.append(field_clause) + params.extend(field_params) + return " AND ".join(f"({clause})" for clause in clauses), params + + def _compile_metadata_field_filter(self, field: str, condition: Any) -> tuple[str, list[Any]]: + if not isinstance(condition, dict) or not any(str(key).startswith("$") for key in condition): + condition = {"$eq": condition} + operator, expected = next(iter(condition.items())) + field_id = self.field_id(field) + if operator == "$eq": + return ( + """ + EXISTS ( + SELECT 1 FROM metadata_values mv + WHERE mv.file_ref = f.file_ref + AND mv.field_id = ? + AND mv.value_text = ? + ) + """, + [field_id, self._metadata_compare_text(expected)], + ) + if operator == "$ne": + return ( + """ + NOT EXISTS ( + SELECT 1 FROM metadata_values mv + WHERE mv.file_ref = f.file_ref + AND mv.field_id = ? + AND mv.value_text = ? + ) + """, + [field_id, self._metadata_compare_text(expected)], + ) + if operator == "$in": + values = list(expected) + if not values: + return "0", [] + placeholders = ", ".join("?" for _ in values) + return ( + f""" + EXISTS ( + SELECT 1 FROM metadata_values mv + WHERE mv.file_ref = f.file_ref + AND mv.field_id = ? + AND mv.value_text IN ({placeholders}) + ) + """, + [field_id, *(self._metadata_compare_text(item) for item in values)], + ) + if operator == "$contains": + return ( + """ + EXISTS ( + SELECT 1 FROM metadata_values mv + WHERE mv.file_ref = f.file_ref + AND mv.field_id = ? + AND lower(mv.value_text) LIKE lower(?) ESCAPE '\\' + ) + """, + [field_id, self._contains_like(self._metadata_compare_text(expected))], + ) + raise ValueError(f"Unsupported metadata operator: {operator}") + + def get_file(self, file_ref: str) -> FileEntry: + with self.connect() as conn: + row = self._file_entry_row(conn, file_ref) + if row is None: + raise KeyError(f"Unknown file_ref: {file_ref}") + return self._file_entry(row) + + def update_file_metadata_status( + self, + file_ref: str, + *, + metadata: dict[str, Any], + metadata_status: dict[str, Any], + ) -> None: + with self.connect() as conn: + row = self._file_entry_row(conn, file_ref) + if row is None: + raise KeyError(f"Unknown file_ref: {file_ref}") + indexed_metadata = self.indexed_metadata_values(metadata) + conn.execute( + """ + UPDATE files + SET metadata_json = ?, + metadata_status_json = ?, + updated_at = CURRENT_TIMESTAMP + WHERE file_ref = ? AND deleted_at IS NULL + """, + ( + json.dumps(metadata, ensure_ascii=False), + json.dumps(metadata_status, ensure_ascii=False), + file_ref, + ), + ) + self.replace_metadata_values( + conn, + file_ref, + indexed_metadata, + ) + + def delete_file(self, target: str) -> None: + with self.connect() as conn: + file_ref = self._resolve_file_ref(conn, target) + conn.execute("DELETE FROM metadata_values WHERE file_ref = ?", (file_ref,)) + conn.execute("DELETE FROM files WHERE file_ref = ?", (file_ref,)) + + def folder_exists(self, path: str) -> bool: + path = normalize_path(path) + with self.connect() as conn: + row = conn.execute( + "SELECT 1 FROM folders WHERE path = ?", + (path,), + ).fetchone() + return row is not None + + def delete_empty_folder(self, path: str) -> bool: + path = normalize_path(path) + if path == "/": + return False + with self.connect() as conn: + folder = self._folder_by_path(conn, path) + if folder is None: + return False + has_files = conn.execute( + """ + SELECT 1 + FROM file_folders + WHERE folder_id = ? + LIMIT 1 + """, + (folder["folder_id"],), + ).fetchone() + if has_files is not None: + return False + has_children = conn.execute( + """ + SELECT 1 + FROM folders + WHERE parent_id = ? + LIMIT 1 + """, + (folder["folder_id"],), + ).fetchone() + if has_children is not None: + return False + conn.execute("DELETE FROM folders WHERE folder_id = ?", (folder["folder_id"],)) + return True + + def resolve_file_ref(self, target: str) -> str: + with self.connect() as conn: + return self._resolve_file_ref(conn, target) + + def _resolve_file_ref(self, conn: sqlite3.Connection, target: str) -> str: + target = str(target).strip() + if not target: + raise KeyError("Empty file target") + row = conn.execute( + "SELECT file_ref FROM files WHERE file_ref = ? AND deleted_at IS NULL", + (target,), + ).fetchone() + if row: + return row["file_ref"] + row = conn.execute( + "SELECT file_ref FROM files WHERE external_id = ? AND deleted_at IS NULL", + (target,), + ).fetchone() + if row: + return row["file_ref"] + virtual_file_ref = self._resolve_virtual_file_ref(conn, target) + if virtual_file_ref: + return virtual_file_ref + raise KeyError(f"Unknown file target: {target}") + + def _resolve_virtual_file_ref(self, conn: sqlite3.Connection, target: str) -> str | None: + virtual_target = normalize_path(target) + rows = conn.execute( + """ + WITH virtual_matches AS ( + SELECT + f.file_ref, + f.external_id, + f.title, + COALESCE( + NULLIF(json_extract(ff.metadata_json, '$.display_name'), ''), + f.title + ) AS display_title, + pf.path AS folder_path, + (CASE WHEN pf.path = '/' THEN '/' ELSE pf.path || '/' END) + || ltrim( + COALESCE( + NULLIF(json_extract(ff.metadata_json, '$.display_name'), ''), + f.title + ), + '/' + ) AS title_virtual_path + FROM files f + JOIN file_folders ff ON ff.file_ref = f.file_ref + JOIN folders pf ON pf.folder_id = ff.folder_id + WHERE f.deleted_at IS NULL + ) + SELECT + file_ref, + external_id, + display_title AS title, + MIN(folder_path) AS folder_path + FROM virtual_matches + WHERE title_virtual_path = ? + GROUP BY file_ref, external_id, display_title + ORDER BY file_ref + LIMIT 2 + """, + (virtual_target,), + ).fetchall() + if not rows: + return None + if len(rows) > 1: + matches = "; ".join(self._virtual_match_summary(row) for row in rows) + raise KeyError(f"Ambiguous file target: {target}. Matches: {matches}") + return rows[0]["file_ref"] + + @staticmethod + def _virtual_match_summary(row: sqlite3.Row) -> str: + external_id = row["external_id"] or "-" + return ( + f"file_ref={row['file_ref']} external_id={external_id} " + f"folder={row['folder_path']} title={row['title']!r}" + ) + + def ensure_folder( + self, + conn: sqlite3.Connection | None, + path: str, + *, + kind: str = "manual", + description: str = "", + metadata: dict[str, Any] | None = None, + ) -> str: + owns_connection = conn is None + if conn is None: + conn = self.connect() + try: + normalized = normalize_path(path) + metadata_json = json.dumps(metadata or {}, ensure_ascii=False) + if normalized == "/": + folder_id = self.folder_id("/") + existing = conn.execute( + "SELECT folder_id FROM folders WHERE path = '/'" + ).fetchone() + if existing is not None and not description and metadata_json == "{}": + if owns_connection: + conn.commit() + return folder_id + self._upsert_folder_row( + conn, + folder_id=folder_id, + parent_id=None, + name="/", + path="/", + kind=kind, + description=description, + metadata_json=metadata_json, + ) + if owns_connection: + conn.commit() + return folder_id + parent_id = self.ensure_folder(conn, str(Path(normalized).parent), kind=kind) + name = normalized.rsplit("/", 1)[-1] + folder_id = self.folder_id(normalized) + self._upsert_folder_row( + conn, + folder_id=folder_id, + parent_id=parent_id, + name=name, + path=normalized, + kind=kind, + description=description, + metadata_json=metadata_json, + ) + if owns_connection: + conn.commit() + return folder_id + finally: + if owns_connection: + conn.close() + + def _upsert_folder_row( + self, + conn: sqlite3.Connection, + *, + folder_id: str, + parent_id: str | None, + name: str, + path: str, + kind: str, + description: str, + metadata_json: str, + ) -> None: + columns = self._columns(conn, "folders") + insert_columns = ["folder_id", "parent_id", "name", "path", "description", "kind", "metadata_json"] + values: list[Any] = [folder_id, parent_id, name, path, description, kind, metadata_json] + if "source" in columns: + insert_columns.append("source") + values.append("system") + if "sort_order" in columns: + insert_columns.append("sort_order") + values.append(0) + placeholders = ", ".join("?" for _ in values) + update_assignments = [ + "parent_id = excluded.parent_id", + "name = excluded.name", + "kind = excluded.kind", + "updated_at = CURRENT_TIMESTAMP", + ] + if description: + update_assignments.append("description = excluded.description") + if metadata_json != "{}": + update_assignments.append("metadata_json = excluded.metadata_json") + conn.execute( + f""" + INSERT INTO folders({", ".join(insert_columns)}) + VALUES ({placeholders}) + ON CONFLICT(path) DO UPDATE SET + {", ".join(update_assignments)} + """, + values, + ) + + def _resolve_or_create_folder(self, conn: sqlite3.Connection, folder_path_or_id: str) -> str: + target = str(folder_path_or_id).strip() + if not target: + raise KeyError("Empty folder target") + row = conn.execute( + "SELECT folder_id FROM folders WHERE folder_id = ?", + (target,), + ).fetchone() + if row: + return row["folder_id"] + row = conn.execute( + "SELECT folder_id FROM folders WHERE path = ?", + (normalize_path(target),), + ).fetchone() + if row: + return row["folder_id"] + return self.ensure_folder(conn, target) + + def read_text(self, file_ref: str) -> str: + entry = self.get_file(file_ref) + return Path(entry.text_artifact_path).read_text(encoding="utf-8") + + def write_text_artifact(self, file_ref: str, content: str) -> Path: + path = self.text_dir / f"{file_ref}.txt" + path.write_text(content, encoding="utf-8") + return path + + def update_pageindex_pointer( + self, + file_ref: str, + *, + pageindex_doc_id: str | None, + pageindex_tree_status: str, + ) -> None: + with self.connect() as conn: + resolved = self._resolve_file_ref(conn, file_ref) + conn.execute( + """ + UPDATE files + SET pageindex_doc_id = ?, + pageindex_tree_status = ?, + updated_at = CURRENT_TIMESTAMP + WHERE file_ref = ? AND deleted_at IS NULL + """, + (pageindex_doc_id, pageindex_tree_status, resolved), + ) + + def write_raw_artifact(self, file_ref: str, metadata: dict[str, Any]) -> Path: + path = self.raw_dir / f"{file_ref}.json" + path.write_text(json.dumps(metadata, ensure_ascii=False, indent=2), encoding="utf-8") + return path + + def file_info(self, target: str) -> dict[str, Any]: + file_ref = self.resolve_file_ref(target) + entry = self.get_file(file_ref) + info = self._file_entry_to_dict(entry) + info["folders"] = self.folder_memberships(file_ref) + return info + + def file_matches( + self, + file_ref: str, + *, + scope: Optional[dict[str, Any]] = None, + metadata_filter: Optional[dict[str, Any]] = None, + ) -> bool: + where = ["f.file_ref = ?", "f.deleted_at IS NULL"] + params: list[Any] = [file_ref] + scope_sql, scope_params = self._scope_sql(scope) + if scope_sql: + where.append(scope_sql) + params.extend(scope_params) + metadata_sql, metadata_params = self._metadata_filter_sql(metadata_filter) + where.extend(metadata_sql) + params.extend(metadata_params) + with self.connect() as conn: + row = conn.execute( + f""" + SELECT 1 + FROM files f + WHERE {" AND ".join(where)} + LIMIT 1 + """, + params, + ).fetchone() + return row is not None + + def folder_memberships(self, file_ref: str) -> list[dict[str, Any]]: + with self.connect() as conn: + rows = conn.execute( + """ + SELECT + fo.folder_id, + fo.parent_id, + fo.name, + fo.path, + fo.description, + fo.kind, + fo.metadata_json AS folder_metadata_json, + ff.metadata_json AS membership_metadata_json, + ff.created_at + FROM file_folders ff + JOIN folders fo ON fo.folder_id = ff.folder_id + WHERE ff.file_ref = ? + ORDER BY fo.path + """, + (file_ref,), + ).fetchall() + return [ + { + "folder_id": row["folder_id"], + "parent_id": row["parent_id"], + "name": row["name"], + "path": row["path"], + "kind": row["kind"], + "description": row["description"], + "folder_metadata": json.loads(row["folder_metadata_json"] or "{}"), + "metadata": json.loads(row["membership_metadata_json"] or "{}"), + "created_at": row["created_at"], + } + for row in rows + ] + + def count_files_in_folder(self, path: str, *, recursive: bool = True) -> int: + path = normalize_path(path) + with self.connect() as conn: + folder = self._folder_by_path(conn, path) + if folder is None: + raise KeyError(f"Unknown folder path: {path}") + if recursive: + row = conn.execute( + """ + SELECT COUNT(DISTINCT f.file_ref) AS count + FROM files f + JOIN file_folders ff ON ff.file_ref = f.file_ref + JOIN folders fo ON fo.folder_id = ff.folder_id + WHERE f.deleted_at IS NULL + AND (fo.path = ? OR fo.path LIKE ? ESCAPE '\\') + """, + (path, self._descendant_like(path)), + ).fetchone() + else: + row = conn.execute( + """ + SELECT COUNT(DISTINCT f.file_ref) AS count + FROM files f + JOIN file_folders ff ON ff.file_ref = f.file_ref + JOIN folders fo ON fo.folder_id = ff.folder_id + WHERE f.deleted_at IS NULL + AND fo.path = ? + """, + (path,), + ).fetchone() + return int(row["count"] or 0) + + def file_basename_exists_in_folder(self, path: str, basename: str) -> bool: + path = normalize_path(path) + basename = str(basename).strip() + if not basename: + return False + with self.connect() as conn: + row = conn.execute( + """ + SELECT 1 + FROM files f + JOIN file_folders ff ON ff.file_ref = f.file_ref + JOIN folders fo ON fo.folder_id = ff.folder_id + WHERE f.deleted_at IS NULL + AND fo.path = ? + AND f.title = ? + LIMIT 1 + """, + ( + path, + basename, + ), + ).fetchone() + return row is not None + + def folder_subtree_thresholds( + self, + path: str, + *, + depth_limit: int, + file_limit: int, + ) -> dict[str, Any]: + path = normalize_path(path) + with self.connect() as conn: + folder = self._folder_by_path(conn, path) + if folder is None: + raise KeyError(f"Unknown folder path: {path}") + base_depth = self._folder_depth(path) + deep_folder = conn.execute( + """ + SELECT path + FROM folders + WHERE path != ? + AND path LIKE ? ESCAPE '\\' + AND ( + CASE + WHEN TRIM(path, '/') = '' THEN 0 + ELSE LENGTH(TRIM(path, '/')) - LENGTH(REPLACE(TRIM(path, '/'), '/', '')) + 1 + END + ) - ? > ? + LIMIT 1 + """, + (path, self._descendant_like(path), base_depth, depth_limit), + ).fetchone() + file_rows = conn.execute( + """ + SELECT DISTINCT f.file_ref + FROM files f + JOIN file_folders ff ON ff.file_ref = f.file_ref + JOIN folders fo ON fo.folder_id = ff.folder_id + WHERE f.deleted_at IS NULL + AND (fo.path = ? OR fo.path LIKE ? ESCAPE '\\') + LIMIT ? + """, + (path, self._descendant_like(path), file_limit + 1), + ).fetchall() + return { + "depth_limit": depth_limit, + "file_limit": file_limit, + "folder_depth_exceeds_limit": deep_folder is not None, + "file_count_exceeds_limit": len(file_rows) > file_limit, + "sampled_file_count": len(file_rows), + "sample_deep_folder_path": deep_folder["path"] if deep_folder is not None else "", + } + + def _file_entry_row(self, conn: sqlite3.Connection, file_ref: str) -> sqlite3.Row | None: + return conn.execute( + """ + SELECT + f.file_ref, + f.external_id, + f.storage_uri, + f.title, + f.descriptor, + f.content_type, + f.source_type, + f.fingerprint, + f.text_artifact_path, + f.raw_artifact_path, + f.pageindex_doc_id, + f.pageindex_tree_status, + f.metadata_json, + f.metadata_status_json, + COALESCE( + ( + SELECT display_folder.path + FROM file_folders display_ff + JOIN folders display_folder + ON display_folder.folder_id = display_ff.folder_id + WHERE display_ff.file_ref = f.file_ref + ORDER BY display_folder.path + LIMIT 1 + ), + '/' + ) AS folder_path + FROM files f + WHERE f.file_ref = ? AND f.deleted_at IS NULL + """, + (file_ref,), + ).fetchone() + + def _file_rows_for_scope( + self, + conn: sqlite3.Connection, + path: str, + recursive: bool, + limit: int, + max_depth: int | None = None, + ) -> list[sqlite3.Row]: + sql = """ + SELECT * + FROM ( + SELECT + f.file_ref, + f.external_id, + f.title, + f.descriptor, + f.pageindex_tree_status, + f.metadata_json, + f.metadata_status_json, + f.created_at, + pf.folder_id AS folder_id, + pf.path AS folder_path, + COALESCE( + NULLIF(json_extract(ff.metadata_json, '$.display_name'), ''), + f.title + ) AS display_title, + ROW_NUMBER() OVER ( + PARTITION BY f.file_ref + ORDER BY pf.path, pf.folder_id + ) AS membership_rank + FROM files f + JOIN file_folders ff ON ff.file_ref = f.file_ref + JOIN folders pf ON pf.folder_id = ff.folder_id + WHERE f.deleted_at IS NULL + """ + params: list[Any] + if recursive: + sql += " AND (pf.path = ? OR pf.path LIKE ? ESCAPE '\\')" + params = [path, self._descendant_like(path)] + if max_depth is not None: + if max_depth <= 0: + sql += " AND 0" + else: + sql += f" AND ({self._folder_depth_sql('pf.path')} - ?) <= ?" + params.extend([self._folder_depth(path), max_depth - 1]) + else: + sql += " AND pf.path = ?" + params = [path] + sql += """ + ) + WHERE membership_rank = 1 + ORDER BY lower(display_title), display_title, lower(folder_path), folder_path, file_ref + LIMIT ? + """ + params.append(limit) + return conn.execute(sql, params).fetchall() + + def _scope_sql(self, scope: Optional[dict[str, Any]]) -> tuple[str, list[Any]]: + if not scope: + return "", [] + recursive = scope.get("recursive", True) + max_depth = scope.get("max_depth") + if max_depth is not None: + max_depth = int(max_depth) + if max_depth < 0: + raise ValueError("max_depth must be non-negative") + folder_id = scope.get("folder_id") + if folder_id: + if folder_id == "root": + folder_path = "/" + else: + if recursive: + if max_depth == 0: + return "0", [] + depth_clause = "" + depth_params: list[Any] = [] + if max_depth is not None: + depth_clause = ( + "AND " + f"({self._folder_depth_sql('scope_folder.path')} - " + f"{self._folder_depth_sql('base_folder.path')}) <= ?" + ) + depth_params = [max_depth - 1] + return ( + f""" + EXISTS ( + SELECT 1 + FROM file_folders scope_ff + JOIN folders scope_folder + ON scope_folder.folder_id = scope_ff.folder_id + JOIN folders base_folder + ON base_folder.folder_id = ? + WHERE scope_ff.file_ref = f.file_ref + AND ( + scope_folder.folder_id = base_folder.folder_id + OR scope_folder.path LIKE {self._descendant_like_sql_expr("base_folder.path")} ESCAPE '\\' + ) + {depth_clause} + ) + """, + [folder_id, *depth_params], + ) + return ( + """ + EXISTS ( + SELECT 1 + FROM file_folders scope_ff + WHERE scope_ff.file_ref = f.file_ref + AND scope_ff.folder_id = ? + ) + """, + [folder_id], + ) + elif scope.get("folder_path") or scope.get("path"): + folder_path = normalize_path(scope.get("folder_path") or scope.get("path")) + else: + return "", [] + if recursive and max_depth == 0: + return "0", [] + path_clause = ( + "(scope_folder.path = ? OR scope_folder.path LIKE ? ESCAPE '\\')" + if recursive + else "scope_folder.path = ?" + ) + params = [folder_path, self._descendant_like(folder_path)] if recursive else [folder_path] + depth_clause = "" + if recursive and max_depth is not None: + depth_clause = f"AND ({self._folder_depth_sql('scope_folder.path')} - ?) <= ?" + params.extend([self._folder_depth(folder_path), max_depth - 1]) + return ( + f""" + EXISTS ( + SELECT 1 + FROM file_folders scope_ff + JOIN folders scope_folder + ON scope_folder.folder_id = scope_ff.folder_id + WHERE scope_ff.file_ref = f.file_ref + AND {path_clause} + {depth_clause} + ) + """, + params, + ) + + def _folder_by_path(self, conn: sqlite3.Connection, path: str) -> sqlite3.Row | None: + return conn.execute( + """ + SELECT + folder_id, + parent_id, + name, + path, + description, + kind, + metadata_json, + created_at, + updated_at + FROM folders + WHERE path = ? + """, + (path,), + ).fetchone() + + @classmethod + def _descendant_like(cls, path: str) -> str: + return "/%" if path == "/" else f"{cls._like_escape(path)}/%" + + @staticmethod + def _descendant_like_sql_expr(path_expr: str) -> str: + escaped_expr = SQLiteFileSystemStore._like_escape_sql_expr(path_expr) + return f"CASE WHEN {path_expr} = '/' THEN '/%' ELSE {escaped_expr} || '/%' END" + + @staticmethod + def _contains_like(value: str) -> str: + return f"%{SQLiteFileSystemStore._like_escape(value)}%" + + @staticmethod + def _like_escape(value: str) -> str: + return ( + value.replace("\\", "\\\\") + .replace("%", "\\%") + .replace("_", "\\_") + ) + + @staticmethod + def _like_escape_sql_expr(value_expr: str) -> str: + return ( + f"replace(replace(replace({value_expr}, '\\', '\\\\'), " + "'%', '\\%'), '_', '\\_')" + ) + + @staticmethod + def _folder_depth(path: str) -> int: + stripped = normalize_path(path).strip("/") + return 0 if not stripped else len(stripped.split("/")) + + @staticmethod + def _folder_depth_sql(path_expr: str) -> str: + return ( + "(CASE " + f"WHEN TRIM({path_expr}, '/') = '' THEN 0 " + f"ELSE LENGTH(TRIM({path_expr}, '/')) " + f"- LENGTH(REPLACE(TRIM({path_expr}, '/'), '/', '')) + 1 " + "END)" + ) + + @classmethod + def _folder_row_to_dict(cls, row: sqlite3.Row) -> dict[str, Any]: + return { + "folder_id": row["folder_id"], + "parent_id": row["parent_id"], + "name": row["name"], + "description": cls._row_value(row, "description", ""), + "path": row["path"], + "kind": row["kind"], + "metadata": json.loads(cls._row_value(row, "metadata_json", "{}") or "{}"), + "created_at": cls._row_value(row, "created_at"), + "updated_at": cls._row_value(row, "updated_at"), + "file_count": cls._row_value(row, "file_count", 0), + "children_count": cls._row_value(row, "children_count", 0), + "matched_files": cls._row_value(row, "matched_files", 0), + } + + @classmethod + def _file_summary(cls, row: sqlite3.Row) -> dict[str, Any]: + display_title = cls._row_value(row, "display_title", row["title"]) + return { + "file_ref": row["file_ref"], + "external_id": row["external_id"], + "title": display_title, + "descriptor": cls._row_value(row, "descriptor", row["title"]), + "pageindex_tree_status": cls._row_value( + row, "pageindex_tree_status", "not_built" + ), + "created_at": cls._row_value(row, "created_at"), + "folder_id": cls._row_value(row, "folder_id"), + "folder_path": row["folder_path"], + "metadata": json.loads(row["metadata_json"] or "{}"), + "metadata_status": json.loads( + cls._row_value(row, "metadata_status_json", "{}") or "{}" + ), + } + + @staticmethod + def _row_value(row: sqlite3.Row, key: str, default: Any = None) -> Any: + return row[key] if key in row.keys() else default + + @staticmethod + def _file_entry(row: sqlite3.Row) -> FileEntry: + return FileEntry( + file_ref=row["file_ref"], + external_id=row["external_id"], + storage_uri=row["storage_uri"], + title=row["title"], + descriptor=row["descriptor"], + content_type=row["content_type"], + source_type=row["source_type"], + fingerprint=row["fingerprint"], + text_artifact_path=row["text_artifact_path"], + raw_artifact_path=row["raw_artifact_path"], + pageindex_doc_id=row["pageindex_doc_id"], + pageindex_tree_status=row["pageindex_tree_status"], + metadata=json.loads(row["metadata_json"] or "{}"), + folder_path=row["folder_path"], + metadata_status=json.loads( + SQLiteFileSystemStore._row_value(row, "metadata_status_json", "{}") or "{}" + ), + ) + + @classmethod + def _file_entry_to_dict(cls, entry: FileEntry) -> dict[str, Any]: + return { + "file_ref": entry.file_ref, + "external_id": entry.external_id, + "title": entry.title, + "descriptor": entry.descriptor, + "content_type": entry.content_type, + "source_type": entry.source_type, + "fingerprint": entry.fingerprint, + "pageindex_doc_id": entry.pageindex_doc_id, + "pageindex_tree_status": entry.pageindex_tree_status, + "metadata": entry.metadata, + "metadata_status": entry.metadata_status, + "folder_path": entry.folder_path, + } + + @staticmethod + def _metadata_value_items(value: Any) -> list[str]: + if value is None: + return [] + if isinstance(value, list): + items = [] + for item in value: + items.extend(SQLiteFileSystemStore._metadata_value_items(item)) + return items + return [SQLiteFileSystemStore._metadata_compare_text(value)] + + @staticmethod + def _metadata_compare_text(value: Any) -> str: + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, (dict, list)): + return json.dumps(value, ensure_ascii=False, sort_keys=True) + return "" if value is None else str(value) + + @staticmethod + def indexed_metadata_values(metadata: dict[str, Any]) -> dict[str, Any]: + return {key: value for key, value in metadata.items() if key != "summary"} + + @staticmethod + def _valid_field_name(name: str) -> bool: + return re.match(r"^[A-Za-z][A-Za-z0-9_]*$", str(name)) is not None + + @staticmethod + def folder_id(path: str) -> str: + normalized = normalize_path(path) + if normalized == "/": + return "folder_root" + digest = hashlib.sha1(normalized.encode("utf-8")).hexdigest()[:16] + return f"folder_{digest}" + + @staticmethod + def field_id(name: str) -> str: + digest = hashlib.sha1(name.encode("utf-8")).hexdigest()[:16] + return f"field_{digest}" + + +def normalize_path(path: str | Path | None) -> str: + if path is None: + return "/" + if str(path).strip().lower() == "root": + return "/" + parts = [part for part in str(path).replace("\\", "/").split("/") if part and part != "."] + if ".." in parts: + raise ValueError("PIFS paths must not contain '..'") + return "/" + "/".join(parts) if parts else "/" + + +def make_file_ref(seed: str) -> str: + digest = hashlib.sha1(seed.encode("utf-8")).hexdigest()[:16] + return f"file_{digest}" + + +def fingerprint(content: str) -> str: + return hashlib.sha256(content.encode("utf-8")).hexdigest() diff --git a/pageindex/filesystem/types.py b/pageindex/filesystem/types.py new file mode 100644 index 000000000..6f933e5f6 --- /dev/null +++ b/pageindex/filesystem/types.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Optional + + +@dataclass(frozen=True) +class FileEntry: + file_ref: str + external_id: Optional[str] + storage_uri: str + title: str + descriptor: str + content_type: str + source_type: Optional[str] + fingerprint: str + text_artifact_path: str + raw_artifact_path: Optional[str] + pageindex_doc_id: Optional[str] + pageindex_tree_status: str + metadata: dict[str, Any] + folder_path: str + metadata_status: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class MetadataField: + name: str + description: str = "" + source: str = "manual" + + +@dataclass(frozen=True) +class PIFSQueryScope: + path: str + folder_path: str + metadata_filter: dict[str, str] = field(default_factory=dict) + metadata_axis: Optional[str] = None diff --git a/pifs b/pifs new file mode 100755 index 000000000..fb2dbc08e --- /dev/null +++ b/pifs @@ -0,0 +1,10 @@ +#!/bin/sh +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) + +if [ -x "$SCRIPT_DIR/.venv/bin/python" ]; then + exec "$SCRIPT_DIR/.venv/bin/python" -m pageindex.filesystem.cli "$@" +fi + +exec python3 -m pageindex.filesystem.cli "$@" diff --git a/requirements.txt b/requirements.txt index ae92bc49f..b5e5df9e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ litellm==1.84.0 -# openai-agents # optional: required for examples/agentic_vectorless_rag_demo.py +# openai-agents==0.17.2 # optional: required for pifs chat/ask and examples/agentic_vectorless_rag_demo.py pymupdf==1.26.4 PyPDF2==3.0.1 python-dotenv==1.2.2 pyyaml==6.0.2 +sqlite-vec>=0.1.9 diff --git a/tests/pifs_markdown_fixture.py b/tests/pifs_markdown_fixture.py new file mode 100644 index 000000000..deda5bcac --- /dev/null +++ b/tests/pifs_markdown_fixture.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + + +class FakePageIndexClient: + def __init__(self): + self.documents: dict[str, dict[str, Any]] = {} + + def index(self, file_path, mode="auto"): + path = Path(file_path) + doc_id = f"pi_{path.stem}_{len(self.documents)}" + text = path.read_text(encoding="utf-8") + self.documents[doc_id] = { + "id": doc_id, + "type": "md", + "path": str(path.resolve()), + "doc_name": path.name, + "doc_description": f"Summary for {path.name}", + "line_count": len(text.splitlines()), + "structure": [{"title": path.stem, "node_id": "0001", "nodes": []}], + "pages": [{"page": 1, "content": text}], + } + return doc_id + + def _ensure_doc_loaded(self, doc_id): + return None + + def get_document_structure(self, doc_id): + import json + + return json.dumps(self.documents[doc_id]["structure"]) + + def get_page_content(self, doc_id, pages): + import json + + return json.dumps(self.documents[doc_id]["pages"]) + + +class FakeEmbeddingClient: + def __init__(self, dimensions: int): + self.dimensions = dimensions + + def embed(self, texts: list[str]) -> list[list[float]]: + return [[1.0, *([0.0] * (self.dimensions - 1))] for _ in texts] + + +def register_markdown( + filesystem, + tmp_path: Path, + external_id: str, + folder_path: str, + *, + title: str | None = None, + text: str | None = None, + metadata: dict[str, Any] | None = None, +): + client = getattr(filesystem, "_test_pageindex_client", None) + if client is None: + client = FakePageIndexClient() + filesystem._test_pageindex_client = client + filesystem._pageindex_client = lambda: client + if filesystem.summary_projection is None: + from pageindex.filesystem.semantic_projection import SummaryProjection + + profile = filesystem._summary_embedding_profile() + filesystem.summary_projection = SummaryProjection( + filesystem.summary_projection_index_dir, + profile=profile, + embedder=FakeEmbeddingClient(profile.dimensions), + create=True, + ) + filename = title if title and title.endswith(".md") else f"{external_id}.md" + source = tmp_path / filename + source.write_text(text or f"{external_id} alpha evidence", encoding="utf-8") + return filesystem.register_file( + storage_uri=source.as_uri(), + folder_path=folder_path, + external_id=external_id, + title=title or filename, + content_type="text/markdown", + content=source.read_text(encoding="utf-8"), + metadata=metadata or {}, + ) diff --git a/tests/test_filesystem_store.py b/tests/test_filesystem_store.py new file mode 100644 index 000000000..d9b726d42 --- /dev/null +++ b/tests/test_filesystem_store.py @@ -0,0 +1,131 @@ +import json + +import pytest + +from tests.pifs_markdown_fixture import register_markdown + + +def test_insert_files_does_not_disable_sqlite_synchronous(tmp_path): + from pageindex.filesystem.store import SQLiteFileSystemStore + + statements = [] + + class RecordingStore(SQLiteFileSystemStore): + def connect(self): + conn = super().connect() + conn.set_trace_callback(statements.append) + return conn + + store = RecordingStore(tmp_path / "workspace") + statements.clear() + + store.insert_files( + [ + { + "file_ref": "ref_report", + "external_id": "doc_report", + "storage_uri": "file:///tmp/report.pdf", + "folder_path": "/documents", + "title": "Report", + "descriptor": "documents/report.pdf", + "content_type": "application/pdf", + "source_type": "documents", + "fingerprint": "fingerprint", + "text_artifact_path": "artifacts/text/ref_report.txt", + "raw_artifact_path": None, + "metadata": {}, + "metadata_json": json.dumps({}), + } + ] + ) + + assert not any( + statement.upper().replace(" ", "") == "PRAGMASYNCHRONOUS=OFF" + for statement in statements + ) + + +def test_register_file_rejects_parent_directory_segments(tmp_path): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem(tmp_path / "workspace") + + with pytest.raises(ValueError, match="must not contain '\\.\\.'"): + filesystem.register_file( + storage_uri="file:///tmp/report.txt", + folder_path="/a/../b", + external_id="doc_bad", + title="Bad", + content="bad body", + ) + + +def test_file_upsert_preserves_created_at(tmp_path): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem(tmp_path / "workspace") + file_ref = register_markdown(filesystem, tmp_path, "doc_report", "/documents", text="version one") + with filesystem.store.connect() as conn: + conn.execute( + "UPDATE files SET created_at = '2001-02-03 04:05:06' WHERE file_ref = ?", + (file_ref,), + ) + register_markdown(filesystem, tmp_path, "doc_report", "/documents", text="version two") + + with filesystem.store.connect() as conn: + row = conn.execute( + "SELECT created_at, storage_uri, deleted_at FROM files WHERE file_ref = ?", + (file_ref,), + ).fetchone() + + assert row["created_at"] == "2001-02-03 04:05:06" + assert row["storage_uri"].endswith("/doc_report.md") + assert row["deleted_at"] is None + + +def test_file_upsert_preserves_soft_delete_tombstone(tmp_path): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem(tmp_path / "workspace") + file_ref = register_markdown(filesystem, tmp_path, "doc_report", "/documents", text="version one") + with filesystem.store.connect() as conn: + conn.execute( + "UPDATE files SET deleted_at = '2001-02-03 04:05:06' WHERE file_ref = ?", + (file_ref,), + ) + register_markdown(filesystem, tmp_path, "doc_report", "/documents", text="version two") + + with filesystem.store.connect() as conn: + row = conn.execute( + "SELECT storage_uri, deleted_at FROM files WHERE file_ref = ?", + (file_ref,), + ).fetchone() + + assert row["storage_uri"].endswith("/doc_report.md") + assert row["deleted_at"] == "2001-02-03 04:05:06" + assert filesystem.store.list_files() == [] + + +def test_listing_uses_one_consistent_folder_membership_row(tmp_path): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem(tmp_path / "workspace") + file_ref = register_markdown( + filesystem, + tmp_path, + "doc_shared", + "/a", + title="Original.md", + text="shared body", + ) + filesystem.store.attach_file_to_folder( + file_ref, + "/b", + metadata={"display_name": "Alpha"}, + ) + + listing = filesystem.store.list_folder("/", recursive=True, limit=10) + row = next(item for item in listing["files"] if item["external_id"] == "doc_shared") + + assert row["folder_path"] == "/a" + assert row["title"] == "Original.md" diff --git a/tests/test_import_surface.py b/tests/test_import_surface.py new file mode 100644 index 000000000..ce52b8ec1 --- /dev/null +++ b/tests/test_import_surface.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import builtins +import importlib +import sys + + +def test_filesystem_import_works_without_eager_optional_dependencies(monkeypatch): + blocked_roots = {"litellm", "openai", "PyPDF2", "pymupdf", "sqlite_vec"} + real_import = builtins.__import__ + + def clear_pageindex_modules() -> None: + for name in list(sys.modules): + if name == "pageindex" or name.startswith("pageindex."): + sys.modules.pop(name, None) + + original_pageindex_modules = { + name: module + for name, module in sys.modules.items() + if name == "pageindex" or name.startswith("pageindex.") + } + + def restore_pageindex_modules() -> None: + clear_pageindex_modules() + sys.modules.update(original_pageindex_modules) + + def import_without_optional_deps(name, globals=None, locals=None, fromlist=(), level=0): + root = name.split(".", 1)[0] + if root in blocked_roots: + raise ModuleNotFoundError(f"No module named '{root}'", name=root) + return real_import(name, globals, locals, fromlist, level) + + clear_pageindex_modules() + try: + with monkeypatch.context() as patch: + patch.setattr(builtins, "__import__", import_without_optional_deps) + + filesystem_module = importlib.import_module("pageindex.filesystem") + from pageindex import PageIndexFileSystem as TopLevelPageIndexFileSystem + from pageindex.filesystem import PageIndexFileSystem + + assert filesystem_module.PageIndexFileSystem is PageIndexFileSystem + assert TopLevelPageIndexFileSystem is PageIndexFileSystem + finally: + restore_pageindex_modules() diff --git a/tests/test_pifs_agent_stream.py b/tests/test_pifs_agent_stream.py new file mode 100644 index 000000000..244e8249b --- /dev/null +++ b/tests/test_pifs_agent_stream.py @@ -0,0 +1,324 @@ +import io +import os +import tempfile +import threading +import unittest +from unittest.mock import patch +from types import SimpleNamespace + +from pydantic import BaseModel, ConfigDict + +from pageindex.filesystem import agent as agent_module +from pageindex.filesystem.agent import ( + AGENT_TOOL_POLICY, + AGENT_SYSTEM_PROMPT, + BASH_TOOL_DESCRIPTION, + PIFSAgentSession, + PIFSAgentStreamObserver, + build_agent_model_settings, + build_pifs_agent_instructions, + normalize_agent_stream_mode, + normalize_reasoning_effort, + normalize_reasoning_summary, + pifs_agent_raw_reasoning_enabled, + serialize_agent_final_output, + should_disable_pifs_agent_tracing, + should_use_openai_compatible_chat_model, +) +from pageindex.filesystem import PageIndexFileSystem + + +class StructuredAnswer(BaseModel): + model_config = ConfigDict(extra="forbid") + + answer: str + document_ids: list[str] + + +class PIFSAgentStreamTest(unittest.TestCase): + def raw_event(self, event_type, delta): + return SimpleNamespace( + type="raw_response_event", + data=SimpleNamespace(type=event_type, delta=delta), + ) + + def test_model_stream_prints_output_and_think_deltas(self): + output = io.StringIO() + stream_log = [] + observer = PIFSAgentStreamObserver("model", stream_log=stream_log, output=output) + + observer.handle_event(self.raw_event("response.reasoning_summary_text.delta", "look up folder")) + observer.handle_event(self.raw_event("response.output_text.delta", '{"answer":')) + observer.handle_event(self.raw_event("response.output_text.delta", '"done"}')) + observer.finish() + + printed = output.getvalue() + self.assertIn("[llm reasoning summary stream]", printed) + self.assertIn("look up folder", printed) + self.assertIn("[llm final output stream]", printed) + self.assertIn('{"answer":"done"}', printed.replace("\n", "")) + self.assertEqual( + stream_log, + [ + {"kind": "output", "text": '{"answer":"done"}'}, + {"kind": "think_summary", "text": "look up folder"}, + ], + ) + + def test_tools_mode_does_not_print_model_text(self): + output = io.StringIO() + stream_log = [] + observer = PIFSAgentStreamObserver("tools", stream_log=stream_log, output=output) + + observer.handle_event(self.raw_event("response.output_text.delta", "hidden from tools mode")) + observer.handle_event(self.raw_event("response.function_call_arguments.delta", '{"command":"tree / -L 1"}')) + observer.emit_tool_call("tree / -L 1") + observer.emit_tool_result(ok=True, output='{"ok": true}', seconds=0.001) + observer.finish() + + printed = output.getvalue() + self.assertNotIn("hidden from tools mode", printed) + self.assertIn("[llm -> pifs command]", printed) + self.assertIn("tree / -L 1", printed) + self.assertIn("[pifs -> llm result preview]", printed) + self.assertIn('{"ok": true}', printed) + self.assertEqual( + stream_log[0], {"kind": "tool_call", "command": "tree / -L 1"} + ) + self.assertEqual(stream_log[1]["kind"], "tool_result") + self.assertEqual( + stream_log[2], + {"kind": "tool_args", "text": '{"command":"tree / -L 1"}'}, + ) + + def test_empty_tool_command_is_not_printed_or_logged(self): + output = io.StringIO() + stream_log = [] + observer = PIFSAgentStreamObserver("tools", stream_log=stream_log, output=output) + + observer.emit_tool_call("") + observer.emit_tool_call(" ") + + self.assertEqual(output.getvalue(), "") + self.assertEqual(stream_log, []) + + def test_tool_result_preview_compacts_large_outputs(self): + output = io.StringIO() + observer = PIFSAgentStreamObserver("tools", output=output) + + observer.emit_tool_result( + ok=True, + output="\n".join(f"line {index}" for index in range(50)), + seconds=0.001, + ) + + printed = output.getvalue() + self.assertIn("[large PIFS result", printed) + self.assertIn("line 0", printed) + self.assertIn("more lines omitted from preview", printed) + self.assertNotIn("line 49", printed) + + def test_raw_reasoning_is_not_logged_by_default_but_summary_is(self): + output = io.StringIO() + stream_log = [] + previous = os.environ.pop("PAGEINDEX_PIFS_AGENT_RAW_REASONING", None) + try: + observer = PIFSAgentStreamObserver("model", stream_log=stream_log, output=output) + observer.handle_event(self.raw_event("response.reasoning_text.delta", "private chain")) + observer.handle_event( + self.raw_event("response.reasoning_summary_text.delta", "visible summary") + ) + observer.finish() + finally: + if previous is not None: + os.environ["PAGEINDEX_PIFS_AGENT_RAW_REASONING"] = previous + + printed = output.getvalue() + self.assertNotIn("private chain", printed) + self.assertIn("visible summary", printed) + self.assertEqual(stream_log, [{"kind": "think_summary", "text": "visible summary"}]) + + def test_raw_reasoning_requires_debug_env_flag(self): + self.assertFalse(pifs_agent_raw_reasoning_enabled({})) + self.assertTrue( + pifs_agent_raw_reasoning_enabled({"PAGEINDEX_PIFS_AGENT_RAW_REASONING": "on"}) + ) + self.assertTrue( + pifs_agent_raw_reasoning_enabled({"PAGEINDEX_PIFS_AGENT_RAW_REASONING": "TRUE"}) + ) + self.assertFalse( + pifs_agent_raw_reasoning_enabled({"PAGEINDEX_PIFS_AGENT_RAW_REASONING": "0"}) + ) + + def test_stream_mode_aliases(self): + self.assertEqual(normalize_agent_stream_mode("think"), "model") + self.assertEqual(normalize_agent_stream_mode("debug"), "all") + self.assertEqual(normalize_agent_stream_mode(""), "off") + with self.assertRaises(ValueError): + normalize_agent_stream_mode("nope") + + def test_reasoning_settings_enable_effort_and_summary(self): + try: + settings = build_agent_model_settings( + reasoning_effort="medium", + reasoning_summary="detailed", + ) + except ModuleNotFoundError as exc: + if exc.name == "agents": + self.skipTest("agents package is not installed") + raise + + self.assertIsNotNone(settings) + self.assertEqual(settings.reasoning.effort, "medium") + self.assertEqual(settings.reasoning.summary, "detailed") + self.assertEqual(settings.verbosity, "low") + + def test_reasoning_effort_defaults_to_visible_summary(self): + try: + settings = build_agent_model_settings(reasoning_effort="low") + except ModuleNotFoundError as exc: + if exc.name == "agents": + self.skipTest("agents package is not installed") + raise + + self.assertIsNotNone(settings) + self.assertEqual(settings.reasoning.effort, "low") + self.assertEqual(settings.reasoning.summary, "auto") + + def test_reasoning_and_base_url_normalization(self): + self.assertEqual(normalize_reasoning_effort("xhigh"), "xhigh") + self.assertIsNone(normalize_reasoning_summary("none")) + self.assertFalse(should_use_openai_compatible_chat_model(None)) + self.assertFalse(should_use_openai_compatible_chat_model("https://api.openai.com/v1/")) + self.assertTrue(should_use_openai_compatible_chat_model("https://example.test/v1")) + with self.assertRaises(ValueError): + normalize_reasoning_effort("maximum") + + def test_tracing_is_disabled_by_default_unless_env_enables_it(self): + self.assertTrue(should_disable_pifs_agent_tracing({})) + self.assertFalse( + should_disable_pifs_agent_tracing({"PAGEINDEX_PIFS_AGENT_TRACING": "1"}) + ) + self.assertFalse( + should_disable_pifs_agent_tracing({"PAGEINDEX_PIFS_AGENT_TRACING": "true"}) + ) + self.assertFalse( + should_disable_pifs_agent_tracing({"PAGEINDEX_PIFS_AGENT_TRACING": "on"}) + ) + self.assertTrue( + should_disable_pifs_agent_tracing({"PAGEINDEX_PIFS_AGENT_TRACING": "0"}) + ) + + def test_structured_agent_output_serializes_to_json(self): + output = serialize_agent_final_output( + StructuredAnswer(answer="done", document_ids=["dsid_1"]) + ) + + self.assertEqual(output, '{"answer":"done","document_ids":["dsid_1"]}') + + def test_prompt_tells_agent_to_use_structure_then_page(self): + self.assertIn( + "run cat --structure before the first cat --page", + AGENT_TOOL_POLICY, + ) + self.assertIn("Use cat --structure before any cat --page", BASH_TOOL_DESCRIPTION) + self.assertIn("Use grep only as a single-document lexical fallback", BASH_TOOL_DESCRIPTION) + + def test_prompt_prefers_canonical_evidence_and_final_consistency(self): + self.assertIn("primary statement, reconciliation table, or official table", AGENT_TOOL_POLICY) + self.assertIn("Do a final consistency check", AGENT_TOOL_POLICY) + self.assertIn("headline number", AGENT_TOOL_POLICY) + self.assertIn("yes/no polarity", AGENT_TOOL_POLICY) + self.assertNotIn("FinanceBench", AGENT_TOOL_POLICY) + + def test_prompt_requires_stat_for_metadata_questions(self): + self.assertIn("Use stat only for identity, metadata, or status questions", AGENT_TOOL_POLICY) + self.assertIn("Do not run stat merely to understand what a document says", AGENT_TOOL_POLICY) + self.assertIn("Use stat only for identity, status, or\nmetadata", BASH_TOOL_DESCRIPTION) + + def test_prompt_routes_topic_retrieval_through_browse_after_folder_exploration(self): + self.assertIn("Start with tree", AGENT_TOOL_POLICY) + self.assertIn('browse ""', AGENT_TOOL_POLICY) + self.assertIn("Use browse -R", BASH_TOOL_DESCRIPTION) + self.assertIn("If browse misses", AGENT_TOOL_POLICY) + self.assertIn("browse -R from a broader folder", AGENT_TOOL_POLICY) + self.assertIn("browse candidates are not final evidence", AGENT_TOOL_POLICY) + self.assertIn("verify the relevant facts with cat or grep", AGENT_TOOL_POLICY) + self.assertIn("cat --structure", AGENT_TOOL_POLICY) + self.assertIn("cat --page", AGENT_TOOL_POLICY) + self.assertIn("Use grep for a selected single file", AGENT_TOOL_POLICY) + self.assertIn("Use grep only as a single-document lexical fallback", BASH_TOOL_DESCRIPTION) + + def test_prompt_allows_recursive_browse_for_low_signal_tree_labels(self): + self.assertIn("Start with tree", AGENT_TOOL_POLICY) + self.assertIn('browse ""', AGENT_TOOL_POLICY) + self.assertIn("low-signal folder or virtual-node labels", AGENT_TOOL_POLICY) + self.assertIn("browse -R", AGENT_TOOL_POLICY) + self.assertIn("low-signal folder or virtual-node labels", BASH_TOOL_DESCRIPTION) + + def test_default_agent_prompts_do_not_suggest_legacy_semantic_commands(self): + prompt_surface = "\n".join( + [AGENT_SYSTEM_PROMPT, BASH_TOOL_DESCRIPTION, AGENT_TOOL_POLICY] + ) + + for old_command in ( + "search-summary", + "search-entity", + "search-relation", + "semantic-grep", + "find --name", + "find --relation", + ): + self.assertNotIn(old_command, prompt_surface) + + def test_prompt_rejects_find_grep_as_exhaustive_search(self): + self.assertIn("folder-wide lexical searches", AGENT_TOOL_POLICY) + self.assertIn("Other shell commands, pipes", AGENT_TOOL_POLICY) + self.assertIn("Only after that persistence protocol may you say the workspace lacks evidence", AGENT_TOOL_POLICY) + + def test_system_prompt_sets_workspace_identity_and_scope(self): + self.assertIn("PageIndex FileSystem Demo Agent", AGENT_SYSTEM_PROMPT) + self.assertIn("VectifyAI Team", AGENT_SYSTEM_PROMPT) + self.assertIn("current PageIndex FileSystem\nworkspace", AGENT_SYSTEM_PROMPT) + self.assertIn("unrelated to the current workspace", AGENT_SYSTEM_PROMPT) + self.assertIn("@field/value", AGENT_TOOL_POLICY) + self.assertIn("@field/value", BASH_TOOL_DESCRIPTION) + self.assertNotIn("@field=value", AGENT_TOOL_POLICY) + self.assertNotIn("@field=value", BASH_TOOL_DESCRIPTION) + self.assertIn("do not answer it as\na general-purpose assistant", AGENT_SYSTEM_PROMPT) + self.assertIn("workspace-related topic question", AGENT_SYSTEM_PROMPT) + self.assertIn("clarify only after the persistence protocol", AGENT_SYSTEM_PROMPT) + self.assertIn("browse for candidate documents before asking", AGENT_TOOL_POLICY) + self.assertIn("Only after that persistence protocol may you say the workspace lacks evidence", AGENT_TOOL_POLICY) + + def test_threaded_runtime_error_is_not_retried_on_fresh_loop(self): + session = object.__new__(PIFSAgentSession) + session.executor = SimpleNamespace() + session.normalized_stream_mode = "off" + session.agent_log = [] + session.max_seconds = None + session.max_turns = 1 + session.session = None + session.agent = object() + + main_thread = threading.get_ident() + run_threads = [] + + def fail_asyncio_run(coro): + coro.close() + run_threads.append(threading.get_ident()) + raise RuntimeError("threaded agent failure") + + with ( + patch.object(agent_module.asyncio, "get_running_loop", return_value=object()), + patch.object(agent_module.asyncio, "run", side_effect=fail_asyncio_run), + ): + with self.assertRaisesRegex(RuntimeError, "threaded agent failure"): + session.run("Question: inspect workspace") + + self.assertEqual(len(run_threads), 1) + self.assertNotEqual(run_threads[0], main_thread) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_pifs_cli.py b/tests/test_pifs_cli.py new file mode 100644 index 000000000..7c2ae441a --- /dev/null +++ b/tests/test_pifs_cli.py @@ -0,0 +1,615 @@ +import builtins +import json +import os +import sys +from pathlib import Path + +import pytest + + +@pytest.fixture(autouse=True) +def isolate_pifs_config(monkeypatch, tmp_path): + monkeypatch.setenv("XDG_CONFIG_HOME", str(tmp_path / "xdg-config")) + + +def pifs_config_path(tmp_path): + return tmp_path / "xdg-config" / "pageindex" / "pifs.json" + + +def test_pifs_config_file_overrides_default_location(monkeypatch, tmp_path): + from pageindex.filesystem import cli + + default_path = pifs_config_path(tmp_path) + override_path = tmp_path / "custom-pifs.json" + default_path.parent.mkdir(parents=True, exist_ok=True) + default_path.write_text( + json.dumps({"workspace": str(tmp_path / "default-workspace")}), + encoding="utf-8", + ) + override_path.write_text( + json.dumps({"workspace": str(tmp_path / "override-workspace")}), + encoding="utf-8", + ) + + monkeypatch.setenv("PIFS_CONFIG_FILE", str(override_path)) + + assert cli._configured_workspace() == str(tmp_path / "override-workspace") + + +class FakeFileSystem: + def __init__(self, workspace, **kwargs): + self.workspace = Path(workspace) + self.kwargs = kwargs + + +def test_cli_workspace_does_not_eagerly_open_projection(monkeypatch, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + + filesystem = cli._filesystem_from_workspace(str(workspace)) + + assert filesystem.workspace == workspace + assert filesystem.kwargs.get("summary_projection_embedding_model") is None + + +def test_cli_workspace_without_projection_index_does_not_require_sqlite_vec( + monkeypatch, tmp_path +): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + real_import = builtins.__import__ + + monkeypatch.delitem(sys.modules, "pageindex.filesystem.semantic_projection", raising=False) + monkeypatch.delitem(sys.modules, "pageindex.filesystem.semantic_index", raising=False) + monkeypatch.delitem(sys.modules, "sqlite_vec", raising=False) + + def block_sqlite_vec(name, globals=None, locals=None, fromlist=(), level=0): + if name.split(".", 1)[0] == "sqlite_vec": + raise ModuleNotFoundError("No module named 'sqlite_vec'", name="sqlite_vec") + return real_import(name, globals, locals, fromlist, level) + + monkeypatch.setattr(builtins, "__import__", block_sqlite_vec) + + filesystem = cli._filesystem_from_workspace(str(workspace)) + + assert filesystem.workspace == workspace + + +def test_cli_workspace_uses_embedding_config(monkeypatch, tmp_path): + from pageindex.filesystem import cli + + config_path = pifs_config_path(tmp_path) + workspace = tmp_path / "workspace" + config_path.parent.mkdir(parents=True, exist_ok=True) + config_path.write_text( + json.dumps( + { + "workspace": str(workspace), + "embedding_model": "gemini-embedding-2-preview", + "embedding_dimensions": 3072, + "embedding_timeout": 12.5, + "embedding_base_url": "https://example.invalid/openai/", + "embedding_api_key": "config-key", + } + ), + encoding="utf-8", + ) + + class ConfiguredFileSystem: + def __init__(self, workspace, **kwargs): + self.workspace = Path(workspace) + self.kwargs = kwargs + + monkeypatch.setenv("PIFS_EMBEDDING_API_KEY", "ignored-env-key") + monkeypatch.setenv("OPENAI_API_KEY", "ignored-openai-key") + monkeypatch.setenv("PIFS_EMBEDDING_BASE_URL", "https://ignored.invalid/") + monkeypatch.setattr(cli, "PageIndexFileSystem", ConfiguredFileSystem) + + filesystem = cli._filesystem_from_workspace(str(workspace)) + + assert filesystem.workspace == workspace + assert filesystem.kwargs == { + "summary_projection_embedding_model": "gemini-embedding-2-preview", + "summary_projection_embedding_dimensions": 3072, + "summary_projection_embedding_timeout": 12.5, + "summary_projection_embedding_base_url": "https://example.invalid/openai/", + "summary_projection_embedding_api_key": "ignored-env-key", + } + assert os.environ["PIFS_EMBEDDING_API_KEY"] == "ignored-env-key" + assert os.environ["PIFS_EMBEDDING_BASE_URL"] == "https://ignored.invalid/" + + +@pytest.mark.parametrize( + ("config_key", "pifs_key", "openai_key", "expected"), + [ + ("config-key", None, None, "config-key"), + ("config-key", "pifs-key", "openai-key", "pifs-key"), + ("config-key", None, "openai-key", "config-key"), + (None, None, "openai-key", "openai-key"), + ], +) +def test_cli_embedding_api_key_precedence( + config_key, pifs_key, openai_key, expected, monkeypatch, tmp_path +): + from pageindex.filesystem import cli + + config_path = pifs_config_path(tmp_path) + config_path.parent.mkdir(parents=True, exist_ok=True) + config = {"workspace": str(tmp_path / "workspace")} + if config_key is not None: + config["embedding_api_key"] = config_key + config_path.write_text(json.dumps(config), encoding="utf-8") + if pifs_key is None: + monkeypatch.delenv("PIFS_EMBEDDING_API_KEY", raising=False) + else: + monkeypatch.setenv("PIFS_EMBEDDING_API_KEY", pifs_key) + if openai_key is None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + else: + monkeypatch.setenv("OPENAI_API_KEY", openai_key) + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + + filesystem = cli._filesystem_from_workspace(str(tmp_path / "workspace")) + + assert filesystem.kwargs["summary_projection_embedding_api_key"] == expected + + +def test_cli_passthrough_invokes_pifs_command_executor(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + executor_instances = [] + + class FakeExecutor: + def __init__(self, filesystem): + self.filesystem = filesystem + self.commands = [] + executor_instances.append(self) + + def execute(self, command): + self.commands.append(command) + return f"executed:{command}" + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "PIFSCommandExecutor", FakeExecutor) + + status = cli.main(["--workspace", str(workspace), "tree", "/documents", "-L", "1"]) + + assert status == 0 + assert capsys.readouterr().out == "executed:tree /documents -L 1\n" + assert len(executor_instances) == 1 + assert executor_instances[0].filesystem.workspace == workspace + assert executor_instances[0].commands == ["tree /documents -L 1"] + + +def test_cli_passthrough_returns_nonzero_for_failed_json_envelope(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + + class FakeExecutor: + def __init__(self, filesystem): + self.filesystem = filesystem + + def execute(self, command): + return json.dumps( + {"success": False, "error": {"message": "bad"}, "next_steps": []} + ) + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "PIFSCommandExecutor", FakeExecutor) + + status = cli.main(["--workspace", str(workspace), "find", "/documents"]) + + assert status == 2 + assert json.loads(capsys.readouterr().out)["success"] is False + + +def test_cli_set_workspace_persists_default(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + config_path = pifs_config_path(tmp_path) + workspace = tmp_path / "workspace" + + status = cli.main(["set", "workspace", str(workspace)]) + + assert status == 0 + output = capsys.readouterr().out + assert f"workspace: {workspace}" in output + assert f"config: {config_path}" in output + assert config_path.read_text(encoding="utf-8") == ( + '{\n "workspace": "' + str(workspace) + '"\n}\n' + ) + + +def test_cli_setmeta_replaces_document_metadata(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + calls = [] + + class FakeSetMetaFileSystem(FakeFileSystem): + def set_metadata(self, target, metadata, *, clear=False): + calls.append((self.workspace, target, metadata, clear)) + return { + "path": "/documents/report.md", + "file_ref": "file_report", + "metadata": metadata, + } + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeSetMetaFileSystem) + + status = cli.main( + [ + "--workspace", + str(workspace), + "setmeta", + "/documents/report.md", + '{"ticker":"AAPL"}', + ] + ) + + assert status == 0 + assert calls == [ + (workspace, "/documents/report.md", {"ticker": "AAPL"}, False) + ] + assert json.loads(capsys.readouterr().out) == { + "document_id": None, + "metadata": {"ticker": "AAPL"}, + "metadata_status": {}, + "path": "/documents/report.md", + "status": None, + "title": None, + } + + +def test_cli_setmeta_clear_uses_empty_object(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + calls = [] + + class FakeSetMetaFileSystem(FakeFileSystem): + def set_metadata(self, target, metadata, *, clear=False): + calls.append((self.workspace, target, metadata, clear)) + return { + "path": "/documents/report.md", + "file_ref": "file_report", + "metadata": metadata, + } + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeSetMetaFileSystem) + + status = cli.main( + ["--workspace", str(workspace), "setmeta", "--clear", "/documents/report.md"] + ) + + assert status == 0 + assert calls == [(workspace, "/documents/report.md", {}, True)] + assert json.loads(capsys.readouterr().out) == { + "document_id": None, + "metadata": {}, + "metadata_status": {}, + "path": "/documents/report.md", + "status": None, + "title": None, + } + + +def test_cli_passthrough_uses_configured_workspace(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + config_path = pifs_config_path(tmp_path) + workspace = tmp_path / "workspace" + executor_instances = [] + monkeypatch.delenv("PIFS_WORKSPACE", raising=False) + + class FakeExecutor: + def __init__(self, filesystem): + self.filesystem = filesystem + self.commands = [] + executor_instances.append(self) + + def execute(self, command): + self.commands.append(command) + return f"executed:{command}" + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "PIFSCommandExecutor", FakeExecutor) + + assert cli.main(["set", "workspace", str(workspace)]) == 0 + capsys.readouterr() + + status = cli.main(["tree", "/documents", "-L", "1"]) + + assert status == 0 + assert capsys.readouterr().out == "executed:tree /documents -L 1\n" + assert executor_instances[0].filesystem.workspace == workspace + + +def test_cli_ask_invokes_agent_with_question(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + agent_calls = [] + + def fake_run_pifs_agent(filesystem, question, **kwargs): + agent_calls.append((filesystem, question, kwargs)) + return "agent answer" + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "run_pifs_agent", fake_run_pifs_agent) + + status = cli.main( + [ + "ask", + "--workspace", + str(workspace), + "--model", + "test-model", + "--stream-mode", + "off", + "--max-turns", + "7", + "--max-seconds", + "3.5", + "--reasoning-effort", + "low", + "--reasoning-summary", + "concise", + "What", + "is", + "inside?", + ] + ) + + assert status == 0 + assert capsys.readouterr().out == "agent answer\n" + filesystem, question, kwargs = agent_calls[0] + assert filesystem.workspace == workspace + assert question == "What is inside?" + assert kwargs == { + "model": "test-model", + "stream_mode": "off", + "max_turns": 7, + "max_seconds": 3.5, + "reasoning_effort": "low", + "reasoning_summary": "concise", + } + + +def test_cli_ask_defaults_to_global_agent_model(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + agent_calls = [] + monkeypatch.delenv("PIFS_AGENT_MODEL", raising=False) + monkeypatch.delenv("PIFS_MODEL", raising=False) + + def fake_run_pifs_agent(filesystem, question, **kwargs): + agent_calls.append(kwargs) + return "agent answer" + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "run_pifs_agent", fake_run_pifs_agent) + + status = cli.main(["ask", "--workspace", str(workspace), "What?"]) + + assert status == 0 + assert capsys.readouterr().out == "agent answer\n" + assert agent_calls[0]["model"] == "gpt-5.4" + + +def test_cli_ask_loads_env_file_before_running_agent(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + env_file = tmp_path / ".env" + env_file.write_text("OPENAI_API_KEY=from-dotenv\n", encoding="utf-8") + agent_keys = [] + + def fake_run_pifs_agent(filesystem, question, **kwargs): + agent_keys.append(os.environ.get("OPENAI_API_KEY")) + return "agent answer" + + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "run_pifs_agent", fake_run_pifs_agent) + + status = cli.main( + [ + "ask", + "--workspace", + str(workspace), + "--env-file", + str(env_file), + "What", + "is", + "inside?", + ] + ) + + assert status == 0 + assert capsys.readouterr().out == "agent answer\n" + assert agent_keys == ["from-dotenv"] + + +def test_cli_chat_runs_one_question_and_exits(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + inputs = iter(["", "Summarize the workspace", "exit"]) + session_instances = [] + session_questions = [] + + class FakeSession: + def __init__(self, filesystem, **kwargs): + self.filesystem = filesystem + self.kwargs = kwargs + session_instances.append(self) + + def run(self, question): + session_questions.append((self, question)) + return f"answer:{question}" + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "PIFSAgentSession", FakeSession) + monkeypatch.setattr("builtins.input", lambda prompt="": next(inputs)) + + status = cli.main(["chat", "--workspace", str(workspace), "--model", "test-model"]) + + assert status == 0 + assert capsys.readouterr().out == "" + assert len(session_instances) == 1 + assert session_instances[0].filesystem.workspace == workspace + assert session_questions == [(session_instances[0], "Summarize the workspace")] + assert session_instances[0].kwargs["model"] == "test-model" + assert session_instances[0].kwargs["stream_mode"] == "all" + + +def test_cli_chat_sanitizes_control_input(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + inputs = iter(["\x12", "he\x7fllo\x1b[A", "exit"]) + agent_calls = [] + + class FakeSession: + def __init__(self, filesystem, **kwargs): + pass + + def run(self, question): + agent_calls.append(question) + return f"answer:{question}" + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "PIFSAgentSession", FakeSession) + monkeypatch.setattr("builtins.input", lambda prompt="": next(inputs)) + + status = cli.main(["chat", "--workspace", str(workspace), "--stream-mode", "off"]) + + assert status == 0 + assert agent_calls == ["hllo"] + assert capsys.readouterr().out == "answer:hllo\n" + + +def test_cli_ask_does_not_reprint_streamed_agent_output(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + + def fake_run_pifs_agent(filesystem, question, **kwargs): + print("streamed answer") + return "returned answer" + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "run_pifs_agent", fake_run_pifs_agent) + + status = cli.main( + [ + "ask", + "--workspace", + str(workspace), + "--stream-mode", + "all", + "What", + "is", + "inside?", + ] + ) + + assert status == 0 + assert capsys.readouterr().out == "streamed answer\n" + + +def test_cli_ask_prints_final_answer_with_tool_stream(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + + def fake_run_pifs_agent(filesystem, question, **kwargs): + print("[tool stream]") + return "FOUND 2 documents" + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "run_pifs_agent", fake_run_pifs_agent) + + status = cli.main( + [ + "ask", + "--workspace", + str(workspace), + "--stream-mode", + "tools", + "What", + "is", + "inside?", + ] + ) + + assert status == 0 + assert capsys.readouterr().out == "[tool stream]\nFOUND 2 documents\n" + + +def test_cli_chat_stream_mode_can_be_overridden(monkeypatch, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + inputs = iter(["Summarize the workspace", "exit"]) + session_kwargs = [] + + class FakeSession: + def __init__(self, filesystem, **kwargs): + session_kwargs.append(kwargs) + + def run(self, question): + return f"answer:{question}" + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "PIFSAgentSession", FakeSession) + monkeypatch.setattr("builtins.input", lambda prompt="": next(inputs)) + + status = cli.main( + [ + "chat", + "--workspace", + str(workspace), + "--stream-mode", + "tools", + ] + ) + + assert status == 0 + assert session_kwargs[0]["stream_mode"] == "tools" + + +def test_cli_chat_reuses_one_agent_session_for_multiple_questions(monkeypatch, capsys, tmp_path): + from pageindex.filesystem import cli + + workspace = tmp_path / "workspace" + inputs = iter(["first", "second", "exit"]) + sessions = [] + + class FakeSession: + def __init__(self, filesystem, **kwargs): + self.questions = [] + sessions.append(self) + + def run(self, question): + self.questions.append(question) + return f"answer:{question}" + + monkeypatch.setattr(cli, "PageIndexFileSystem", FakeFileSystem) + monkeypatch.setattr(cli, "PIFSAgentSession", FakeSession) + monkeypatch.setattr("builtins.input", lambda prompt="": next(inputs)) + + status = cli.main(["chat", "--workspace", str(workspace), "--stream-mode", "off"]) + + assert status == 0 + assert len(sessions) == 1 + assert sessions[0].questions == ["first", "second"] + assert capsys.readouterr().out == "answer:first\nanswer:second\n" diff --git a/tests/test_pifs_like_escape.py b/tests/test_pifs_like_escape.py new file mode 100644 index 000000000..262306c70 --- /dev/null +++ b/tests/test_pifs_like_escape.py @@ -0,0 +1,108 @@ +from pathlib import Path + +from tests.pifs_markdown_fixture import register_markdown + + +def _register_file( + filesystem, + tmp_path: Path, + filename: str, + *, + folder_path: str, + external_id: str, + metadata: dict[str, str] | None = None, +) -> None: + register_markdown( + filesystem, + tmp_path, + external_id, + folder_path, + title=f"{external_id}.md", + text=f"{external_id} fixture text", + metadata=metadata, + ) + + +def test_descendant_folder_filter_treats_underscore_literally(tmp_path): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem(workspace=tmp_path / "workspace") + _register_file( + filesystem, + tmp_path, + "literal.txt", + folder_path="/proj_1/docs", + external_id="literal_underscore", + ) + _register_file( + filesystem, + tmp_path, + "wildcard.txt", + folder_path="/projA1/docs", + external_id="wildcard_neighbor", + ) + + recursive = filesystem.store.list_folder("/proj_1", recursive=True, limit=10) + scoped_results = filesystem.store.list_files( + scope={"folder_path": "/proj_1", "recursive": True}, + limit=10, + ) + ranked_folders = {folder["path"]: folder for folder in filesystem.store.find_folders("/", max_depth=1, limit=10)} + + assert {folder["path"] for folder in recursive["folders"]} == {"/proj_1/docs"} + assert {file["external_id"] for file in recursive["files"]} == {"literal_underscore"} + assert {result["external_id"] for result in scoped_results} == {"literal_underscore"} + assert ranked_folders["/proj_1"]["matched_files"] == 1 + assert ranked_folders["/projA1"]["matched_files"] == 1 + assert filesystem.store.count_files_in_folder("/proj_1", recursive=True) == 1 + + +def test_metadata_contains_treats_percent_and_underscore_literally(tmp_path): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem(workspace=tmp_path / "workspace") + filesystem.metadata.register_schema({"fields": {"status": {}}}) + _register_file( + filesystem, + tmp_path, + "percent.txt", + folder_path="/documents", + external_id="literal_percent", + metadata={"status": "100% done"}, + ) + _register_file( + filesystem, + tmp_path, + "percent-neighbor.txt", + folder_path="/documents", + external_id="percent_neighbor", + metadata={"status": "100X done"}, + ) + _register_file( + filesystem, + tmp_path, + "underscore.txt", + folder_path="/documents", + external_id="literal_underscore", + metadata={"status": "build_alpha"}, + ) + _register_file( + filesystem, + tmp_path, + "underscore-neighbor.txt", + folder_path="/documents", + external_id="underscore_neighbor", + metadata={"status": "buildXalpha"}, + ) + + percent_results = filesystem.store.list_files( + metadata_filter={"status": {"$contains": "100% done"}}, + limit=10, + ) + underscore_results = filesystem.store.list_files( + metadata_filter={"status": {"$contains": "build_alpha"}}, + limit=10, + ) + + assert {result["external_id"] for result in percent_results} == {"literal_percent"} + assert {result["external_id"] for result in underscore_results} == {"literal_underscore"} diff --git a/tests/test_pifs_path_resolution.py b/tests/test_pifs_path_resolution.py new file mode 100644 index 000000000..04a70d782 --- /dev/null +++ b/tests/test_pifs_path_resolution.py @@ -0,0 +1,39 @@ +import pytest + +from tests.pifs_markdown_fixture import register_markdown + + +def test_root_virtual_file_path_resolves_without_double_slash(tmp_path): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem(workspace=tmp_path / "workspace") + file_ref = register_markdown( + filesystem, tmp_path, "doc_root_title", "/", title="Root Title.md", text="root content" + ) + + assert filesystem.store.resolve_file_ref("/Root Title.md") == file_ref + + +def test_nested_virtual_file_path_resolves_by_folder_and_title(tmp_path): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem(workspace=tmp_path / "workspace") + first_ref = register_markdown(filesystem, tmp_path, "doc_first", "/a", title="First.md", text="first content") + second_ref = register_markdown(filesystem, tmp_path, "doc_second", "/a/b", title="file.md", text="second content") + + assert filesystem.store.resolve_file_ref("/a/b/file.md") == second_ref + + assert first_ref != second_ref + + +def test_unknown_virtual_file_target_raises_clear_error(tmp_path): + from pageindex.filesystem import PageIndexFileSystem + + filesystem = PageIndexFileSystem(workspace=tmp_path / "workspace") + first_ref = register_markdown(filesystem, tmp_path, "doc_first", "/first", title="First.md", text="first content") + second_ref = register_markdown(filesystem, tmp_path, "doc_second", "/second", title="Second.md", text="second content") + + with pytest.raises(KeyError, match="Unknown file target"): + filesystem.store.resolve_file_ref("/shared/missing.txt") + + assert first_ref != second_ref diff --git a/tests/test_pifs_simplification_acceptance.py b/tests/test_pifs_simplification_acceptance.py new file mode 100644 index 000000000..3708f67b2 --- /dev/null +++ b/tests/test_pifs_simplification_acceptance.py @@ -0,0 +1,2839 @@ +import json +import hashlib +import sqlite3 +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +class FakeOpenAI: + def __init__(self, **_kwargs): + self.embeddings = self + + def create(self, *, model, input, dimensions): + assert model == "test-embedding" + assert dimensions == 3 + + def vector(text): + lowered = str(text).lower() + if "alpha" in lowered: + return [1.0, 0.0, 0.0] + if "beta" in lowered: + return [0.0, 1.0, 0.0] + return [0.0, 0.0, 1.0] + + return SimpleNamespace( + data=[ + SimpleNamespace(index=index, embedding=vector(text)) + for index, text in enumerate(input) + ] + ) + + +def install_network_fakes(monkeypatch): + import openai + from pageindex import PageIndexClient + + def fake_index(self, file_path, mode="auto"): + path = Path(file_path) + text = ( + path.read_text(encoding="utf-8") + if path.suffix.lower() in {".md", ".markdown"} + else "pdf alpha evidence" + ) + path_digest = hashlib.sha256(str(path.resolve()).encode("utf-8")).hexdigest() + document_id = f"pageindex_{path_digest[:16]}" + document = { + "id": document_id, + "type": "pdf" if path.suffix.lower() == ".pdf" else "md", + "path": str(path.resolve()), + "doc_name": path.name, + "doc_description": f"Summary for {path.name}: {text}", + "line_count": len(text.splitlines()), + "structure": [ + { + "title": path.stem, + "node_id": "0001", + "line_num": 1, + "text": text, + "nodes": [], + } + ], + "pages": [{"page": 1, "content": text}], + } + self.documents[document_id] = document + if self.workspace: + self._save_doc(document_id) + return document_id + + monkeypatch.setattr(openai, "OpenAI", FakeOpenAI) + monkeypatch.setattr(PageIndexClient, "index", fake_index) + + +def write_embedding_config(tmp_path, monkeypatch): + config = tmp_path / "pifs.json" + config.write_text( + json.dumps( + { + "embedding_base_url": "https://EXAMPLE.invalid/v1/", + "embedding_model": "test-embedding", + "embedding_dimensions": 3, + "embedding_timeout": 12, + } + ), + encoding="utf-8", + ) + monkeypatch.setenv("PIFS_CONFIG_FILE", str(config)) + monkeypatch.setenv("PIFS_EMBEDDING_API_KEY", "runtime-secret") + return config + + +def logical_tables(connection): + return { + row[1] + for row in connection.execute("PRAGMA table_list") + if row[2] in {"table", "virtual"} and not row[1].startswith("sqlite_") + and not row[1].startswith("semantic_index_vec_") + } + + +def workspace_state(workspace): + workspace = Path(workspace) + if not workspace.exists(): + return {} + return { + path.relative_to(workspace).as_posix(): ( + f"symlink:{path.readlink()}" + if path.is_symlink() + else "directory" + if path.is_dir() + else hashlib.sha256(path.read_bytes()).hexdigest() + ) + for path in sorted(workspace.rglob("*")) + } + + +def create_projection_v2(workspace): + from pageindex.filesystem.semantic_projection import ( + SummaryEmbeddingProfile, + SummaryProjection, + ) + + projection_dir = Path(workspace) / "artifacts" / "projection_indexes" + SummaryProjection( + projection_dir, + profile=SummaryEmbeddingProfile( + base_url="https://example.invalid/v1", + model="test-embedding", + dimensions=3, + api_key="runtime-only", + ), + create=True, + ) + return projection_dir + + +def connect_summary_database(path): + import sqlite_vec + + connection = sqlite3.connect(path) + connection.enable_load_extension(True) + sqlite_vec.load(connection) + connection.enable_load_extension(False) + return connection + + +def insert_projection_document(summary_path, *, file_ref="orphan_projection_ref"): + import sqlite_vec + + with connect_summary_database(summary_path) as connection: + connection.execute( + """ + INSERT INTO semantic_index_docs( + rowid, file_ref, external_id, source_type, title, + text_hash, text_chars, metadata_json + ) VALUES (1, ?, 'orphan-doc', 'markdown', 'orphan.md', + 'orphan-hash', 14, '{}') + """, + (file_ref,), + ) + connection.execute( + "INSERT INTO semantic_index_vec(rowid, source_type, embedding) " + "VALUES (1, 'markdown', ?)", + (sqlite_vec.serialize_float32([1.0, 0.0, 0.0]),), + ) + + +def delete_projection_document(summary_path, file_ref): + with connect_summary_database(summary_path) as connection: + row = connection.execute( + "SELECT rowid FROM semantic_index_docs WHERE file_ref = ?", + (file_ref,), + ).fetchone() + assert row is not None + connection.execute( + "DELETE FROM semantic_index_vec WHERE rowid = ?", + (row[0],), + ) + connection.execute( + "DELETE FROM semantic_index_docs WHERE rowid = ?", + (row[0],), + ) + + +def catalog_file_count(workspace): + with sqlite3.connect(Path(workspace) / "filesystem.sqlite") as connection: + return connection.execute("SELECT COUNT(*) FROM files").fetchone()[0] + + +def registration_logical_state(workspace): + workspace = Path(workspace) + catalog_path = workspace / "filesystem.sqlite" + projection_dir = workspace / "artifacts" / "projection_indexes" + summary_path = projection_dir / "summary.sqlite" + cache_path = projection_dir / "embedding_cache.sqlite" + + with sqlite3.connect(catalog_path) as connection: + catalog = { + "files": connection.execute( + """ + SELECT file_ref, external_id, storage_uri, title, descriptor, + content_type, source_type, fingerprint, text_artifact_path, + raw_artifact_path, pageindex_doc_id, pageindex_tree_status, + metadata_json, metadata_status_json, deleted_at + FROM files + ORDER BY file_ref + """ + ).fetchall(), + "folders": connection.execute( + """ + SELECT folder_id, parent_id, name, path, description, kind, metadata_json + FROM folders + ORDER BY path + """ + ).fetchall(), + "metadata_fields": connection.execute( + """ + SELECT field_id, name, description, source + FROM metadata_fields + ORDER BY name + """ + ).fetchall(), + "metadata_values": connection.execute( + """ + SELECT file_ref, field_id, value_text + FROM metadata_values + ORDER BY file_ref, field_id, value_text + """ + ).fetchall(), + } + + summary = {"docs": [], "vec": []} + if summary_path.is_file(): + with connect_summary_database(summary_path) as connection: + summary = { + "docs": connection.execute( + """ + SELECT rowid, file_ref, external_id, source_type, title, + text_hash, text_chars, metadata_json + FROM semantic_index_docs + ORDER BY rowid + """ + ).fetchall(), + "vec": connection.execute( + """ + SELECT rowid, source_type, hex(embedding) + FROM semantic_index_vec + ORDER BY rowid + """ + ).fetchall(), + } + + cache = [] + if cache_path.is_file(): + with sqlite3.connect(cache_path) as connection: + cache = connection.execute( + """ + SELECT base_url, model, dimensions, text_hash, hex(vector_blob) + FROM embedding_cache + ORDER BY base_url, model, dimensions, text_hash + """ + ).fetchall() + + artifacts = {} + for artifact_kind in ("text", "raw"): + artifact_dir = workspace / "artifacts" / artifact_kind + if artifact_dir.is_dir(): + artifacts[artifact_kind] = { + path.relative_to(artifact_dir).as_posix(): hashlib.sha256( + path.read_bytes() + ).hexdigest() + for path in sorted(artifact_dir.rglob("*")) + if path.is_file() + } + else: + artifacts[artifact_kind] = {} + + pageindex_dir = workspace / "artifacts" / "pageindex_client" + meta_path = pageindex_dir / "_meta.json" + artifacts["pageindex_client"] = { + "meta": json.loads(meta_path.read_text(encoding="utf-8")) + if meta_path.is_file() + else {}, + "documents": { + path.name: { + "json": json.loads(path.read_text(encoding="utf-8")), + "sha256": hashlib.sha256(path.read_bytes()).hexdigest(), + } + for path in sorted(pageindex_dir.glob("*.json")) + if path.name != "_meta.json" + }, + } + + return { + "catalog": catalog, + "summary": summary, + "cache": cache, + "artifacts": artifacts, + } + + +def open_test_filesystem(workspace): + from pageindex.filesystem import PageIndexFileSystem + + return PageIndexFileSystem( + workspace, + summary_projection_embedding_base_url="https://example.invalid/v1", + summary_projection_embedding_model="test-embedding", + summary_projection_embedding_dimensions=3, + summary_projection_embedding_api_key="runtime-secret", + ) + + +def nested_path_with_length(root, target_length): + path = Path(root) + while len(str(path)) < target_length: + component_length = min(200, target_length - len(str(path)) - 1) + path /= "x" * component_length + assert len(str(path)) == target_length + return path + + +def test_cli_rejects_removed_ls_command_with_structured_error(tmp_path, capsys): + from pageindex.filesystem.cli import main + + status = main(["--workspace", str(tmp_path / "workspace"), "ls", "/"]) + + payload = json.loads(capsys.readouterr().out) + assert status == 2 + assert payload == { + "success": False, + "error": {"code": "invalid_command", "message": "Unsupported command: ls"}, + "next_steps": [], + } + + +def test_fresh_cli_workspace_uses_the_five_table_catalog_schema_v2(tmp_path, capsys): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + assert json.loads(capsys.readouterr().out)["success"] is True + + with sqlite3.connect(workspace / "filesystem.sqlite") as connection: + version = connection.execute("PRAGMA user_version").fetchone()[0] + tables = { + row[0] + for row in connection.execute( + "SELECT name FROM sqlite_master " + "WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" + ) + } + + assert version == 2 + assert tables == { + "files", + "folders", + "file_folders", + "metadata_fields", + "metadata_values", + } + + +def test_cli_rejects_legacy_catalog_without_mutating_it(tmp_path, capsys): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + workspace.mkdir() + database = workspace / "filesystem.sqlite" + with sqlite3.connect(database) as connection: + connection.execute("CREATE TABLE legacy_sentinel(value TEXT NOT NULL)") + connection.execute("INSERT INTO legacy_sentinel(value) VALUES ('preserve me')") + connection.execute("PRAGMA user_version = 1") + before = hashlib.sha256(database.read_bytes()).hexdigest() + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert hashlib.sha256(database.read_bytes()).hexdigest() == before + + +@pytest.mark.parametrize( + "mutation", + ["extra_column", "missing_index", "missing_primary_and_foreign_keys"], +) +def test_cli_rejects_pseudo_v2_catalog_without_mutating_workspace( + mutation, tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + database = workspace / "filesystem.sqlite" + with sqlite3.connect(database) as connection: + if mutation == "extra_column": + connection.execute("ALTER TABLE files ADD COLUMN legacy_provider TEXT") + elif mutation == "missing_index": + connection.execute("DROP INDEX idx_files_external_id") + else: + connection.executescript( + """ + ALTER TABLE file_folders RENAME TO legacy_file_folders; + CREATE TABLE file_folders ( + file_ref TEXT NOT NULL, + folder_id TEXT NOT NULL, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT DEFAULT CURRENT_TIMESTAMP + ); + DROP TABLE legacy_file_folders; + CREATE INDEX idx_file_folders_folder ON file_folders(folder_id); + """ + ) + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + + +def test_cli_rejects_partial_legacy_projection_without_creating_summary( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + projection_dir = workspace / "artifacts" / "projection_indexes" + projection_dir.mkdir(parents=True) + cache_path = projection_dir / "embedding_cache.sqlite" + with sqlite3.connect(cache_path) as connection: + connection.execute( + "CREATE TABLE embedding_cache(provider TEXT, model TEXT, text_hash TEXT)" + ) + connection.execute("PRAGMA user_version = 1") + before = hashlib.sha256(cache_path.read_bytes()).hexdigest() + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 1 + + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert hashlib.sha256(cache_path.read_bytes()).hexdigest() == before + assert not (projection_dir / "summary.sqlite").exists() + assert catalog_file_count(workspace) == 0 + + +def test_cli_preflights_partial_projection_before_creating_catalog(tmp_path, capsys): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + projection_dir = workspace / "artifacts" / "projection_indexes" + projection_dir.mkdir(parents=True) + cache_path = projection_dir / "embedding_cache.sqlite" + with sqlite3.connect(cache_path) as connection: + connection.execute( + "CREATE TABLE embedding_cache(provider TEXT, model TEXT, text_hash TEXT)" + ) + connection.execute("PRAGMA user_version = 1") + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + assert not (workspace / "filesystem.sqlite").exists() + + +@pytest.mark.parametrize("catalog_state", ["missing", "zero_byte"]) +def test_cli_rejects_projection_pair_without_valid_catalog_before_mutating_workspace( + catalog_state, tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + create_projection_v2(workspace) + if catalog_state == "zero_byte": + (workspace / "filesystem.sqlite").write_bytes(b"") + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + if catalog_state == "missing": + assert not (workspace / "filesystem.sqlite").exists() + else: + assert (workspace / "filesystem.sqlite").stat().st_size == 0 + + +@pytest.mark.parametrize( + "projection_state", + ["both_broken", "broken_summary_only", "broken_summary_with_valid_cache"], +) +def test_cli_rejects_broken_projection_symlinks_without_mutating_workspace( + projection_state, tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + projection_dir = workspace / "artifacts" / "projection_indexes" + if projection_state == "broken_summary_with_valid_cache": + create_projection_v2(workspace) + (projection_dir / "summary.sqlite").unlink() + else: + projection_dir.mkdir(parents=True) + (projection_dir / "summary.sqlite").symlink_to("missing-summary.sqlite") + if projection_state == "both_broken": + (projection_dir / "embedding_cache.sqlite").symlink_to( + "missing-cache.sqlite" + ) + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + + +def test_cli_rejects_existing_zero_byte_catalog_without_projection_or_mutation( + tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + workspace.mkdir() + catalog = workspace / "filesystem.sqlite" + catalog.write_bytes(b"") + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + assert catalog.stat().st_size == 0 + + +@pytest.mark.parametrize("mutation", ["orphan_projection", "missing_root"]) +def test_cli_rejects_inconsistent_catalog_projection_relationships_without_mutation( + mutation, tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + projection_dir = create_projection_v2(workspace) + insert_projection_document(projection_dir / "summary.sqlite") + if mutation == "missing_root": + with sqlite3.connect(workspace / "filesystem.sqlite") as connection: + connection.execute("DELETE FROM folders WHERE path = '/'") + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + + +@pytest.mark.parametrize( + ("catalog_state", "projection_state", "should_fail"), + [ + ("active", "projected", False), + ("active", "missing", True), + ("deleted", "missing", False), + ("deleted", "projected", True), + ], +) +def test_runtime_requires_complete_projection_for_every_active_catalog_file( + catalog_state, projection_state, should_fail, tmp_path, monkeypatch +): + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + source = tmp_path / "notes.md" + source.write_text("alpha consistency evidence", encoding="utf-8") + filesystem = open_test_filesystem(workspace) + file_ref = filesystem.register_file( + storage_uri=source.as_uri(), + folder_path="/documents", + title=source.name, + content_type="text/markdown", + content=source.read_text(encoding="utf-8"), + ) + if catalog_state == "deleted": + with sqlite3.connect(workspace / "filesystem.sqlite") as connection: + connection.execute( + "UPDATE files SET deleted_at = '2026-01-02 03:04:05' " + "WHERE file_ref = ?", + (file_ref,), + ) + if projection_state == "missing": + delete_projection_document( + workspace / "artifacts" / "projection_indexes" / "summary.sqlite", + file_ref, + ) + before = workspace_state(workspace) + + if should_fail: + with pytest.raises(RuntimeError, match="migrate_pifs_workspace.py"): + open_test_filesystem(workspace) + else: + open_test_filesystem(workspace) + + assert workspace_state(workspace) == before + + +@pytest.mark.parametrize(("catalog_state", "should_fail"), [("active", True), ("deleted", False)]) +def test_runtime_requires_projection_pair_when_catalog_only_has_active_files( + catalog_state, should_fail, tmp_path, monkeypatch +): + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + source = tmp_path / "notes.md" + source.write_text("alpha catalog-only evidence", encoding="utf-8") + filesystem = open_test_filesystem(workspace) + file_ref = filesystem.register_file( + storage_uri=source.as_uri(), + folder_path="/documents", + title=source.name, + content_type="text/markdown", + content=source.read_text(encoding="utf-8"), + ) + if catalog_state == "deleted": + with sqlite3.connect(workspace / "filesystem.sqlite") as connection: + connection.execute( + "UPDATE files SET deleted_at = '2026-01-02 03:04:05' " + "WHERE file_ref = ?", + (file_ref,), + ) + projection_dir = workspace / "artifacts" / "projection_indexes" + (projection_dir / "summary.sqlite").unlink() + (projection_dir / "embedding_cache.sqlite").unlink() + before = workspace_state(workspace) + + if should_fail: + with pytest.raises(RuntimeError, match="migrate_pifs_workspace.py"): + open_test_filesystem(workspace) + else: + open_test_filesystem(workspace) + + assert workspace_state(workspace) == before + + +@pytest.mark.parametrize( + "mutation", + ["missing_vector", "extra_vector", "source_type_mismatch", "wrong_blob_length"], +) +def test_runtime_rejects_projection_document_vector_mismatches_without_mutation( + mutation, tmp_path, monkeypatch +): + import sqlite_vec + + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + source = tmp_path / "notes.md" + source.write_text("alpha vector consistency evidence", encoding="utf-8") + filesystem = open_test_filesystem(workspace) + filesystem.register_file( + storage_uri=source.as_uri(), + folder_path="/documents", + title=source.name, + content_type="text/markdown", + content=source.read_text(encoding="utf-8"), + ) + summary_path = workspace / "artifacts" / "projection_indexes" / "summary.sqlite" + with connect_summary_database(summary_path) as connection: + row = connection.execute( + "SELECT rowid, source_type, embedding FROM semantic_index_vec" + ).fetchone() + assert row is not None + if mutation == "missing_vector": + connection.execute( + "DELETE FROM semantic_index_vec WHERE rowid = ?", (row[0],) + ) + elif mutation == "extra_vector": + connection.execute( + "INSERT INTO semantic_index_vec(rowid, source_type, embedding) " + "VALUES (999, 'markdown', ?)", + (sqlite_vec.serialize_float32([0.0, 0.0, 1.0]),), + ) + elif mutation == "source_type_mismatch": + connection.execute( + "DELETE FROM semantic_index_vec WHERE rowid = ?", (row[0],) + ) + connection.execute( + "INSERT INTO semantic_index_vec(rowid, source_type, embedding) " + "VALUES (?, 'pdf', ?)", + (row[0], row[2]), + ) + else: + connection.execute( + "UPDATE semantic_index_vec_vector_chunks00 SET vectors = zeroblob(4)" + ) + before = workspace_state(workspace) + + with pytest.raises(RuntimeError, match="migrate this workspace"): + open_test_filesystem(workspace) + + assert workspace_state(workspace) == before + + +@pytest.mark.parametrize( + "mutation", + [ + "summary_extra_column", + "summary_missing_primary_and_unique", + "summary_missing_index", + "summary_extra_config_key", + "summary_vec_dimension_mismatch", + "cache_extra_column", + "cache_missing_primary_key", + ], +) +def test_cli_rejects_pseudo_v2_projection_without_mutating_workspace( + mutation, tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + projection_dir = create_projection_v2(workspace) + summary_path = projection_dir / "summary.sqlite" + cache_path = projection_dir / "embedding_cache.sqlite" + if mutation.startswith("summary_"): + with sqlite3.connect(summary_path) as connection: + if mutation == "summary_extra_column": + connection.execute( + "ALTER TABLE semantic_index_docs ADD COLUMN legacy_provider TEXT" + ) + elif mutation == "summary_missing_primary_and_unique": + connection.executescript( + """ + DROP INDEX idx_semantic_index_docs_external_id; + DROP INDEX idx_semantic_index_docs_source_type; + ALTER TABLE semantic_index_docs RENAME TO legacy_semantic_index_docs; + CREATE TABLE semantic_index_docs ( + rowid INTEGER, + file_ref TEXT NOT NULL, + external_id TEXT, + source_type TEXT NOT NULL DEFAULT '', + title TEXT NOT NULL DEFAULT '', + text_hash TEXT NOT NULL, + text_chars INTEGER NOT NULL DEFAULT 0, + metadata_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT DEFAULT CURRENT_TIMESTAMP, + updated_at TEXT DEFAULT CURRENT_TIMESTAMP + ); + DROP TABLE legacy_semantic_index_docs; + CREATE INDEX idx_semantic_index_docs_external_id + ON semantic_index_docs(external_id); + CREATE INDEX idx_semantic_index_docs_source_type + ON semantic_index_docs(source_type); + """ + ) + elif mutation == "summary_missing_index": + connection.execute("DROP INDEX idx_semantic_index_docs_external_id") + elif mutation == "summary_extra_config_key": + connection.execute( + "INSERT INTO semantic_index_config(key, value) " + "VALUES ('legacy_provider', 'openai')" + ) + else: + connection.execute( + "UPDATE semantic_index_config SET value = '4' WHERE key = 'dimension'" + ) + connection.execute( + "UPDATE semantic_index_config SET value = ? WHERE key = 'metadata'", + ( + json.dumps( + { + "base_url": "https://example.invalid/v1", + "model": "test-embedding", + "dimensions": 4, + } + ), + ), + ) + else: + with sqlite3.connect(cache_path) as connection: + if mutation == "cache_extra_column": + connection.execute( + "ALTER TABLE embedding_cache ADD COLUMN provider TEXT" + ) + else: + connection.executescript( + """ + ALTER TABLE embedding_cache RENAME TO legacy_embedding_cache; + CREATE TABLE embedding_cache ( + base_url TEXT NOT NULL, + model TEXT NOT NULL, + dimensions INTEGER NOT NULL CHECK(dimensions > 0), + text_hash TEXT NOT NULL, + vector_blob BLOB NOT NULL, + created_at TEXT DEFAULT CURRENT_TIMESTAMP + ); + DROP TABLE legacy_embedding_cache; + """ + ) + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + assert not (workspace / "filesystem.sqlite").exists() + + +def test_cli_rejects_noncanonical_vec0_distance_without_mutating_workspace( + tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + projection_dir = create_projection_v2(workspace) + with connect_summary_database(projection_dir / "summary.sqlite") as connection: + connection.execute("DROP TABLE semantic_index_vec") + connection.execute( + "CREATE VIRTUAL TABLE semantic_index_vec USING " + "vec0(source_type TEXT partition key, " + "embedding float[3] distance_metric=cosine)" + ) + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + + +def test_cli_accepts_canonical_vec0_declaration_with_spacing_and_case_variation( + tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + projection_dir = create_projection_v2(workspace) + with connect_summary_database(projection_dir / "summary.sqlite") as connection: + connection.execute("DROP TABLE semantic_index_vec") + connection.execute( + "CREATE VIRTUAL TABLE semantic_index_vec USING " + "VEC0( source_type TEXT PARTITION KEY , embedding FLOAT [ 3 ] )" + ) + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 0 + assert json.loads(capsys.readouterr().out)["success"] is True + assert workspace_state(workspace) == before + + +@pytest.mark.parametrize( + "identity_case", + [ + "summary_base_url", + "summary_model", + "cache_base_url", + "cache_model", + ], +) +def test_cli_rejects_noncanonical_persisted_embedding_identity_without_mutation( + identity_case, tmp_path, capsys +): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + projection_dir = create_projection_v2(workspace) + if identity_case.startswith("summary_"): + with sqlite3.connect(projection_dir / "summary.sqlite") as connection: + metadata = json.loads( + connection.execute( + "SELECT value FROM semantic_index_config WHERE key = 'metadata'" + ).fetchone()[0] + ) + if identity_case == "summary_base_url": + metadata["base_url"] = "HTTPS://EXAMPLE.INVALID/v1/" + else: + metadata["model"] = " test-embedding " + connection.execute( + "UPDATE semantic_index_config SET value = ? WHERE key = 'metadata'", + (json.dumps(metadata, sort_keys=True),), + ) + else: + base_url = ( + "HTTPS://EXAMPLE.INVALID/v1/" + if identity_case == "cache_base_url" + else "https://example.invalid/v1" + ) + model = ( + " test-embedding " + if identity_case == "cache_model" + else "test-embedding" + ) + with sqlite3.connect(projection_dir / "embedding_cache.sqlite") as connection: + connection.execute( + "INSERT INTO embedding_cache(" + "base_url, model, dimensions, text_hash, vector_blob" + ") VALUES (?, ?, 3, 'cache-hash', ?)", + (base_url, model, b"\0" * 12), + ) + before = workspace_state(workspace) + + status = main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) + + assert status == 1 + assert "migrate_pifs_workspace.py" in capsys.readouterr().err + assert workspace_state(workspace) == before + + +def test_cli_allows_fresh_listing_when_projection_is_completely_absent(tmp_path, capsys): + from pageindex.filesystem.cli import main + + workspace = tmp_path / "workspace" + + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + assert json.loads(capsys.readouterr().out)["success"] is True + assert (workspace / "filesystem.sqlite").is_file() + assert not (workspace / "artifacts" / "projection_indexes").exists() + + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + assert json.loads(capsys.readouterr().out)["success"] is True + assert not (workspace / "artifacts" / "projection_indexes").exists() + + +def test_pageindex_is_the_only_public_python_entry_point_for_pifs(): + import pageindex + import pageindex.filesystem as filesystem_module + + assert pageindex.PageIndexFileSystem is filesystem_module.PageIndexFileSystem + for internal_name in ( + "PIFSCommandExecutor", + "OpenResult", + "SearchResult", + "SummaryProjectionIndexer", + "SemanticProjectionSearchBackend", + "SQLiteVecSemanticIndex", + "SemanticIndexRecord", + "SemanticSearchResult", + ): + assert not hasattr(filesystem_module, internal_name) + assert not hasattr(filesystem_module, "_LAZY_EXPORTS") + + +def test_cli_add_creates_migration_compatible_summary_and_cache_v2( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + config = write_embedding_config(tmp_path, monkeypatch) + config_values = json.loads(config.read_text(encoding="utf-8")) + config_values["embedding_api_key"] = "config-secret" + config.write_text(json.dumps(config_values), encoding="utf-8") + monkeypatch.delenv("PIFS_EMBEDDING_API_KEY") + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + output = capsys.readouterr().out + + projection_dir = workspace / "artifacts" / "projection_indexes" + with sqlite3.connect(projection_dir / "summary.sqlite") as summary: + summary_version = summary.execute("PRAGMA user_version").fetchone()[0] + summary_tables = logical_tables(summary) + metadata = json.loads( + summary.execute( + "SELECT value FROM semantic_index_config WHERE key = 'metadata'" + ).fetchone()[0] + ) + with sqlite3.connect(projection_dir / "embedding_cache.sqlite") as cache: + cache_version = cache.execute("PRAGMA user_version").fetchone()[0] + cache_tables = logical_tables(cache) + cache_columns = { + row[1] for row in cache.execute("PRAGMA table_info(embedding_cache)") + } + + assert summary_version == cache_version == 2 + assert summary_tables == { + "semantic_index_config", + "semantic_index_docs", + "semantic_index_vec", + } + assert cache_tables == {"embedding_cache"} + assert cache_columns == { + "base_url", + "model", + "dimensions", + "text_hash", + "vector_blob", + "created_at", + } + assert metadata == { + "base_url": "https://example.invalid/v1", + "model": "test-embedding", + "dimensions": 3, + } + for database in ( + workspace / "filesystem.sqlite", + projection_dir / "summary.sqlite", + projection_dir / "embedding_cache.sqlite", + ): + assert b"config-secret" not in database.read_bytes() + assert "config-secret" not in output + + +def test_cli_add_reports_path_without_persistence_identity(tmp_path, monkeypatch, capsys): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + + assert ( + main( + [ + "--workspace", + str(tmp_path / "workspace"), + "add", + str(source), + "/documents", + ] + ) + == 0 + ) + + output = capsys.readouterr().out + assert output == "added: /documents/notes.md\n" + + +def test_cli_browse_reopens_owned_file_with_canonical_result_fields( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + source.unlink() + + assert ( + main( + [ + "--workspace", + str(workspace), + "browse", + "/documents", + "alpha", + ] + ) + == 0 + ) + + payload = json.loads(capsys.readouterr().out) + assert payload["success"] is True + document = payload["data"]["documents"][0] + assert set(document) == { + "path", + "document_id", + "title", + "status", + "rank", + "similarity", + "summary", + "metadata", + "folder_path", + "folder_paths", + } + assert document["path"] == "/documents/notes.md" + assert document["title"] == "notes.md" + assert document["status"] == "built" + assert document["rank"] == 1 + assert document["summary"].startswith("Summary for notes.md") + assert document["folder_path"] == "/documents" + + +def test_cli_stat_translates_persistence_identity_once(tmp_path, monkeypatch, capsys): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + + assert main(["--workspace", str(workspace), "stat", "/documents/notes.md"]) == 0 + + document = json.loads(capsys.readouterr().out)["data"]["document"] + assert set(document) == { + "path", + "document_id", + "title", + "status", + "content_type", + "metadata", + "metadata_status", + "folder_paths", + } + assert document["path"] == "/documents/notes.md" + assert document["title"] == "notes.md" + assert document["status"] == "built" + assert document["folder_paths"] == ["/documents"] + + +def test_cli_cat_reads_structure_without_internal_identity(tmp_path, monkeypatch, capsys): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + source.unlink() + + assert ( + main( + [ + "--workspace", + str(workspace), + "cat", + "/documents/notes.md", + "--structure", + ] + ) + == 0 + ) + + data = json.loads(capsys.readouterr().out)["data"] + assert set(data["document"]) == { + "path", + "document_id", + "title", + "status", + "content_type", + "metadata", + "metadata_status", + "folder_paths", + "available", + } + assert data["structure"] == [ + {"title": "notes", "node_id": "0001", "line_num": 1, "nodes": []} + ] + + +def create_metadata_scope_cli_fixture(tmp_path, monkeypatch, capsys): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + records = [ + ( + "current.md", + "alpha current evidence", + "/documents", + {"year": 2024, "ticker": "AAPL", "sector": "finance/tech"}, + ), + ( + "filing.md", + "alpha filing evidence", + "/documents/sec-filings", + {"year": 2024, "ticker": "AAPL", "doc_type": "10-K"}, + ), + ( + "prior.md", + "alpha prior evidence", + "/documents", + {"year": 2023, "ticker": "MSFT"}, + ), + ("archive.md", "archive evidence", "/archive", {"region": "EMEA"}), + ] + for filename, content, folder, metadata in records: + source = tmp_path / filename + source.write_text(content, encoding="utf-8") + assert main(["--workspace", str(workspace), "add", str(source), folder]) == 0 + capsys.readouterr() + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + f"{folder}/{filename}", + json.dumps(metadata), + ] + ) == 0 + capsys.readouterr() + return workspace + + +def test_tree_folder_trailing_slash_lists_scope_local_metadata_axes( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + workspace = create_metadata_scope_cli_fixture(tmp_path, monkeypatch, capsys) + + payloads = [] + for folder_path in ("/documents", "/documents/"): + assert main( + ["--workspace", str(workspace), "tree", folder_path, "-L", "1"] + ) == 0 + payloads.append(json.loads(capsys.readouterr().out)) + + assert payloads[1]["data"] == payloads[0]["data"] + tree = payloads[0]["data"]["tree"] + assert tree["path"] == "/documents" + axes = { + row["name"]: row + for row in tree["folders"] + if row["type"] == "metadata_axis" + } + + assert set(axes) == {"@doc_type", "@sector", "@ticker", "@year"} + assert {name: row["path"] for name, row in axes.items()} == { + "@doc_type": "/documents/@doc_type", + "@sector": "/documents/@sector", + "@ticker": "/documents/@ticker", + "@year": "/documents/@year", + } + assert {row["type"] for row in axes.values()} == {"metadata_axis"} + physical = [row for row in tree["folders"] if row["type"] == "folder"] + assert [(row["name"], row["path"]) for row in physical] == [ + ("sec-filings", "/documents/sec-filings") + ] + assert all(row["type"] != "file" for row in tree["folders"]) + assert "@region" not in axes + + +def test_tree_metadata_axis_trailing_slash_lists_actionable_paginated_values( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + workspace = create_metadata_scope_cli_fixture(tmp_path, monkeypatch, capsys) + + payloads = [] + for axis_path in ("/documents/@year", "/documents/@year/"): + assert main( + ["--workspace", str(workspace), "tree", axis_path, "-L", "1"] + ) == 0 + payloads.append(json.loads(capsys.readouterr().out)) + + values = payloads[0]["data"]["tree"]["folders"] + assert payloads[1]["data"] == payloads[0]["data"] + assert [(row["name"], row["type"], row["path"]) for row in values] == [ + ("2024", "metadata_value", "/documents/@year/2024"), + ("2023", "metadata_value", "/documents/@year/2023"), + ] + assert payloads[0]["data"]["pagination"] == { + "page": 1, + "page_size": 50, + "has_more": False, + "next_page": None, + } + + selected = values[0]["path"] + scoped_payloads = [] + for value_path in (selected, f"{selected}/"): + assert main( + ["--workspace", str(workspace), "tree", value_path, "-L", "1"] + ) == 0 + scoped_payloads.append(json.loads(capsys.readouterr().out)) + assert scoped_payloads[1]["data"] == scoped_payloads[0]["data"] + selected_tree = scoped_payloads[0]["data"]["tree"] + assert selected_tree["path"] == selected + assert {row["title"] for row in selected_tree["files"]} == { + "current.md", + "filing.md", + } + selected_folders = { + (row["type"], row["name"]): row["path"] + for row in selected_tree["folders"] + } + assert selected_folders == { + ("folder", "sec-filings"): "/documents/sec-filings/@year/2024", + ("metadata_axis", "@doc_type"): "/documents/@year/2024/@doc_type", + ("metadata_axis", "@sector"): "/documents/@year/2024/@sector", + ("metadata_axis", "@ticker"): "/documents/@year/2024/@ticker", + } + assert main( + ["--workspace", str(workspace), "browse", selected, "alpha"] + ) == 0 + documents = json.loads(capsys.readouterr().out)["data"]["documents"] + assert {row["title"] for row in documents} == {"current.md", "filing.md"} + + locator = selected_tree["files"][0]["path"] + for command in ( + ["stat", locator], + ["cat", locator, "--structure"], + ["grep", "alpha", locator], + ): + assert main(["--workspace", str(workspace), *command]) == 0 + capsys.readouterr() + + child_scope = selected_folders[("folder", "sec-filings")] + assert main(["--workspace", str(workspace), "tree", child_scope, "-L", "1"]) == 0 + child_tree = json.loads(capsys.readouterr().out)["data"]["tree"] + assert [row["title"] for row in child_tree["files"]] == ["filing.md"] + + assert main( + ["--workspace", str(workspace), "tree", "/documents/@sector/", "-L", "1"] + ) == 0 + encoded = json.loads(capsys.readouterr().out)["data"]["tree"]["folders"][0] + assert encoded["path"] == "/documents/@sector/finance%2Ftech" + assert main( + ["--workspace", str(workspace), "tree", encoded["path"], "-L", "1"] + ) == 0 + capsys.readouterr() + + +def test_tree_metadata_values_keep_fixed_fifty_item_pagination( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "tickers.md" + source.write_text("alpha ticker evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + tickers = [f"T{index:02d}" for index in range(55)] + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + "/documents/tickers.md", + json.dumps({"ticker": tickers}), + ] + ) == 0 + capsys.readouterr() + + assert main( + ["--workspace", str(workspace), "tree", "/documents/@ticker/", "-L", "1"] + ) == 0 + first = json.loads(capsys.readouterr().out)["data"] + assert [row["value"] for row in first["tree"]["folders"]] == tickers[:50] + assert first["pagination"] == { + "page": 1, + "page_size": 50, + "has_more": True, + "next_page": 2, + } + + assert main( + [ + "--workspace", + str(workspace), + "tree", + "/documents/@ticker", + "--page", + "2", + ] + ) == 0 + second = json.loads(capsys.readouterr().out)["data"] + assert [row["value"] for row in second["tree"]["folders"]] == tickers[50:] + assert second["pagination"] == { + "page": 2, + "page_size": 50, + "has_more": False, + "next_page": None, + } + + +def test_tree_metadata_dot_values_return_actionable_encoded_scope_paths( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha dot value evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + "/documents/notes.md", + json.dumps({"tag": [".", ".."]}), + ] + ) == 0 + capsys.readouterr() + + assert main(["--workspace", str(workspace), "tree", "/documents/@tag"]) == 0 + values = json.loads(capsys.readouterr().out)["data"]["tree"]["folders"] + assert {row["value"]: row["path"] for row in values} == { + ".": "/documents/@tag/%2E", + "..": "/documents/@tag/%2E%2E", + } + + for scope_path in (row["path"] for row in values): + assert main( + ["--workspace", str(workspace), "tree", scope_path, "-L", "1"] + ) == 0 + file_path = json.loads(capsys.readouterr().out)["data"]["tree"]["files"][0][ + "path" + ] + assert main( + ["--workspace", str(workspace), "browse", scope_path, "alpha"] + ) == 0 + document = json.loads(capsys.readouterr().out)["data"]["documents"][0] + assert document["path"] == file_path + assert main(["--workspace", str(workspace), "stat", file_path]) == 0 + capsys.readouterr() + + +def test_metadata_virtual_paths_use_alternating_encoded_field_value_segments( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + assert ( + main( + [ + "--workspace", + str(workspace), + "setmeta", + "/documents/notes.md", + '{"year": 2024, "sector": "finance/tech"}', + ] + ) + == 0 + ) + capsys.readouterr() + + assert main(["--workspace", str(workspace), "tree", "/documents/@year"]) == 0 + year_payload = json.loads(capsys.readouterr().out) + values = year_payload["data"]["tree"]["folders"] + assert values[0]["path"] == "/documents/@year/2024" + assert year_payload["next_steps"] == [ + 'browse /documents/@year/ ""' + ] + + assert ( + main( + [ + "--workspace", + str(workspace), + "tree", + "/documents/@year/2024", + "-L", + "1", + ] + ) + == 0 + ) + scoped_tree = json.loads(capsys.readouterr().out)["data"]["tree"] + assert scoped_tree["files"][0]["path"] == "/documents/@year/2024/notes.md" + assert [folder["path"] for folder in scoped_tree["folders"]] == [ + "/documents/@year/2024/@sector" + ] + + assert main( + [ + "--workspace", + str(workspace), + "tree", + "/documents/@year/2024/@sector", + ] + ) == 0 + sector = json.loads(capsys.readouterr().out)["data"]["tree"]["folders"][0] + assert sector["path"] == "/documents/@year/2024/@sector/finance%2Ftech" + + locator = f"{sector['path']}/notes.md" + for command in ( + ["stat", locator], + ["cat", locator, "--structure"], + ["grep", "alpha", locator], + ): + assert main(["--workspace", str(workspace), *command]) == 0 + capsys.readouterr() + + assert main( + ["--workspace", str(workspace), "tree", "/documents/@year=2024"] + ) == 2 + error = json.loads(capsys.readouterr().out)["error"] + assert "@field/value" in error["message"] + + +def test_setmeta_translates_identity_at_the_cli_boundary( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + "/documents/notes.md", + '{"year": 2024}', + ] + ) == 0 + document = json.loads(capsys.readouterr().out) + + assert set(document) == { + "path", + "document_id", + "title", + "status", + "metadata", + "metadata_status", + } + assert document["path"] == "/documents/notes.md" + assert document["metadata"]["year"] == 2024 + + +def test_setmeta_by_file_ref_returns_an_actionable_path_without_leaking_identity( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + with sqlite3.connect(workspace / "filesystem.sqlite") as connection: + file_ref = connection.execute("SELECT file_ref FROM files").fetchone()[0] + + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + file_ref, + '{"year": 2024}', + ] + ) == 0 + output = capsys.readouterr().out + document = json.loads(output) + + assert document["path"] == "/documents/notes.md" + assert file_ref not in output + assert main(["--workspace", str(workspace), "stat", document["path"]]) == 0 + stat = json.loads(capsys.readouterr().out)["data"]["document"] + assert stat["path"] == document["path"] + + +@pytest.mark.parametrize("operation", ["replace", "clear"]) +def test_metadata_scoped_setmeta_returns_the_post_update_actionable_path( + operation, tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + "/documents/notes.md", + '{"year": 2024}', + ] + ) == 0 + capsys.readouterr() + old_path = "/documents/@year/2024/notes.md" + command = ["--workspace", str(workspace), "setmeta"] + if operation == "clear": + command.extend(["--clear", old_path]) + else: + command.extend([old_path, '{"year": 2025}']) + + assert main(command) == 0 + document = json.loads(capsys.readouterr().out) + + assert document["path"] == "/documents/notes.md" + assert document["path"] != old_path + if operation == "clear": + assert "year" not in document["metadata"] + else: + assert document["metadata"]["year"] == 2025 + assert main(["--workspace", str(workspace), "stat", document["path"]]) == 0 + stat = json.loads(capsys.readouterr().out)["data"]["document"] + assert stat["path"] == document["path"] + + +def test_tree_file_records_use_only_command_identity_names( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + + assert main(["--workspace", str(workspace), "tree", "/documents", "-L", "1"]) == 0 + file_record = json.loads(capsys.readouterr().out)["data"]["tree"]["files"][0] + + assert set(file_record) == { + "path", + "document_id", + "title", + "status", + "type", + "metadata", + } + + +def test_duplicate_virtual_leaves_use_actionable_paths_without_internal_ids( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + for folder_name in ("a", "b"): + source_dir = tmp_path / folder_name + source_dir.mkdir() + source = source_dir / "notes.md" + source.write_text(f"alpha evidence {folder_name}", encoding="utf-8") + folder = f"/documents/{folder_name}" + assert main(["--workspace", str(workspace), "add", str(source), folder]) == 0 + capsys.readouterr() + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + f"{folder}/notes.md", + '{"year": 2024}', + ] + ) == 0 + capsys.readouterr() + + assert main( + ["--workspace", str(workspace), "tree", "/documents/@year/2024", "-L", "1"] + ) == 0 + files = json.loads(capsys.readouterr().out)["data"]["tree"]["files"] + paths = [row["path"] for row in files] + + assert len(paths) == len(set(paths)) == 2 + assert all("file_" not in path for path in paths) + for path in paths: + assert main(["--workspace", str(workspace), "stat", path]) == 0 + assert json.loads(capsys.readouterr().out)["data"]["document"]["path"] == path + + +@pytest.mark.parametrize( + ("filename", "content"), + [ + ("notes.md", b"markdown alpha evidence"), + ("report.pdf", b"%PDF-1.4 fake fixture"), + ], +) +def test_cli_add_owns_supported_documents_and_keeps_evidence_readable( + filename, content, tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / filename + source.write_bytes(content) + workspace = tmp_path / "workspace" + + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + source.unlink() + + owned = list((workspace / "artifacts" / "uploads").glob(f"*/{filename}")) + assert len(owned) == 1 + assert owned[0].read_bytes() == content + assert main( + ["--workspace", str(workspace), "grep", "alpha", f"/documents/{filename}"] + ) == 0 + matches = json.loads(capsys.readouterr().out)["data"]["matches"] + assert matches[0]["text"].endswith("alpha evidence") + + +def test_cli_add_rejects_unsupported_type_before_registration( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.txt" + source.write_text("unsupported", encoding="utf-8") + workspace = tmp_path / "workspace" + + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 1 + + assert "Unsupported file type" in capsys.readouterr().err + assert catalog_file_count(workspace) == 0 + assert not list((workspace / "artifacts" / "uploads").glob("**/*")) + + +def test_cli_add_rolls_back_when_pageindex_fails( + tmp_path, monkeypatch, capsys +): + from pageindex import PageIndexClient + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + + def fail_index(self, file_path, mode="auto"): + raise RuntimeError("pageindex unavailable") + + monkeypatch.setattr(PageIndexClient, "index", fail_index) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 1 + + assert "pageindex unavailable" in capsys.readouterr().err + assert catalog_file_count(workspace) == 0 + assert not list((workspace / "artifacts" / "uploads").glob("**/*")) + pageindex_cache = workspace / "artifacts" / "pageindex_client" + assert json.loads((pageindex_cache / "_meta.json").read_text(encoding="utf-8")) == {} + assert not [path for path in pageindex_cache.glob("*.json") if path.name != "_meta.json"] + + +def test_cli_add_rolls_back_when_summary_projection_fails( + tmp_path, monkeypatch, capsys +): + import openai + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + + class FailingOpenAI(FakeOpenAI): + def create(self, *, model, input, dimensions): + raise RuntimeError("embedding unavailable") + + monkeypatch.setattr(openai, "OpenAI", FailingOpenAI) + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 1 + + assert "summary projection" in capsys.readouterr().err.lower() + assert catalog_file_count(workspace) == 0 + assert not list((workspace / "artifacts" / "uploads").glob("**/*")) + with sqlite3.connect( + workspace / "artifacts" / "projection_indexes" / "summary.sqlite" + ) as connection: + assert connection.execute( + "SELECT COUNT(*) FROM semantic_index_docs" + ).fetchone()[0] == 0 + + +@pytest.mark.parametrize("cache_state", ["fresh", "preexisting_shared_key"]) +def test_add_restores_embedding_cache_when_vec_upsert_fails( + cache_state, tmp_path, monkeypatch +): + from pageindex.filesystem.semantic_index import SQLiteVecSemanticIndex + + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + filesystem = open_test_filesystem(workspace) + source_dir = tmp_path / "add" + source_dir.mkdir() + source = source_dir / "notes.md" + source.write_text("alpha shared cache evidence", encoding="utf-8") + + if cache_state == "preexisting_shared_key": + existing_dir = tmp_path / "existing" + existing_dir.mkdir() + existing = existing_dir / "notes.md" + existing.write_text(source.read_text(encoding="utf-8"), encoding="utf-8") + filesystem.register_file( + storage_uri=existing.as_uri(), + folder_path="/documents/existing", + title=existing.name, + content_type="text/markdown", + content=existing.read_text(encoding="utf-8"), + ) + else: + create_projection_v2(workspace) + filesystem = open_test_filesystem(workspace) + + baseline = registration_logical_state(workspace) + baseline_uploads = sorted( + path.relative_to(workspace).as_posix() + for path in workspace.glob("artifacts/uploads/**/*") + ) + + def fail_vec_upsert(self, records): + raise RuntimeError("vec upsert unavailable after cache write") + + monkeypatch.setattr(SQLiteVecSemanticIndex, "upsert_many", fail_vec_upsert) + + with pytest.raises(RuntimeError, match="vec upsert unavailable after cache write"): + filesystem.add_file(source, "/documents/new") + + assert registration_logical_state(workspace) == baseline + assert sorted( + path.relative_to(workspace).as_posix() + for path in workspace.glob("artifacts/uploads/**/*") + ) == baseline_uploads + reopened = open_test_filesystem(workspace) + assert registration_logical_state(workspace) == baseline + if cache_state == "preexisting_shared_key": + assert reopened.store.file_refs_for_scope() == filesystem.store.file_refs_for_scope() + else: + assert reopened.store.file_refs_for_scope() == [] + + +def test_register_file_completes_summary_projection_for_immediate_and_reopen_browse( + tmp_path, monkeypatch +): + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + source = tmp_path / "notes.md" + source.write_text("alpha registration evidence", encoding="utf-8") + filesystem = open_test_filesystem(workspace) + caller_metadata = {"year": 2024, "labels": {"team": "alpha"}} + caller_metadata_baseline = json.loads(json.dumps(caller_metadata)) + + file_ref = filesystem.register_file( + storage_uri=source.as_uri(), + folder_path="/documents", + title=source.name, + content_type="text/markdown", + content=source.read_text(encoding="utf-8"), + metadata=caller_metadata, + ) + + projection_dir = workspace / "artifacts" / "projection_indexes" + assert (projection_dir / "summary.sqlite").is_file() + assert (projection_dir / "embedding_cache.sqlite").is_file() + stored = filesystem.store.get_file(file_ref) + assert caller_metadata == caller_metadata_baseline + assert "summary" not in caller_metadata + assert stored.metadata["summary"].startswith("Summary for notes.md:") + assert stored.metadata_status["summary_projection"]["status"] == "ready" + assert file_ref in { + row["file_ref"] + for row in filesystem.browse_semantic_files("/documents", "alpha")["data"] + } + + reopened = open_test_filesystem(workspace) + assert file_ref in { + row["file_ref"] + for row in reopened.browse_semantic_files("/documents", "alpha")["data"] + } + + +def test_register_file_opens_existing_projection_and_upserts_second_document( + tmp_path, monkeypatch +): + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + first = tmp_path / "first.md" + second = tmp_path / "second.md" + first.write_text("alpha registration evidence", encoding="utf-8") + second.write_text("beta registration evidence", encoding="utf-8") + filesystem = open_test_filesystem(workspace) + first_ref = filesystem.register_file( + storage_uri=first.as_uri(), + folder_path="/documents", + title=first.name, + content_type="text/markdown", + content=first.read_text(encoding="utf-8"), + ) + + reopened = open_test_filesystem(workspace) + second_ref = reopened.register_file( + storage_uri=second.as_uri(), + folder_path="/documents", + title=second.name, + content_type="text/markdown", + content=second.read_text(encoding="utf-8"), + ) + + assert reopened.store.get_file(second_ref).metadata_status["summary_projection"]["status"] == "ready" + assert second_ref in { + row["file_ref"] + for row in reopened.browse_semantic_files("/documents", "beta")["data"] + } + with connect_summary_database( + workspace / "artifacts" / "projection_indexes" / "summary.sqlite" + ) as connection: + assert connection.execute("SELECT COUNT(*) FROM semantic_index_docs").fetchone()[0] == 2 + assert connection.execute("SELECT COUNT(*) FROM semantic_index_vec").fetchone()[0] == 2 + assert set(reopened.store.file_refs_for_scope()) == {first_ref, second_ref} + + +def test_register_files_rolls_back_new_batch_when_second_projection_upsert_fails( + tmp_path, monkeypatch +): + import openai + + install_network_fakes(monkeypatch) + + class FailingSecondOpenAI(FakeOpenAI): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.calls = 0 + + def create(self, *, model, input, dimensions): + self.calls += 1 + if self.calls == 2: + raise RuntimeError("embedding unavailable for second registration") + return super().create(model=model, input=input, dimensions=dimensions) + + monkeypatch.setattr(openai, "OpenAI", FailingSecondOpenAI) + workspace = tmp_path / "workspace" + filesystem = open_test_filesystem(workspace) + first = tmp_path / "first.md" + second = tmp_path / "second.md" + first.write_text("alpha registration evidence", encoding="utf-8") + second.write_text("beta registration evidence", encoding="utf-8") + baseline = registration_logical_state(workspace) + + with pytest.raises(RuntimeError, match="summary projection.*second registration"): + filesystem.register_files( + [ + { + "storage_uri": source.as_uri(), + "folder_path": "/documents/new", + "title": source.name, + "content_type": "text/markdown", + "content": source.read_text(encoding="utf-8"), + "metadata": {"year": 2024}, + } + for source in (first, second) + ] + ) + + assert registration_logical_state(workspace) == baseline + + reopened = open_test_filesystem(workspace) + assert reopened.store.folder_info("/")["path"] == "/" + assert reopened.store.file_refs_for_scope() == [] + + +def test_register_files_restores_existing_document_when_later_projection_upsert_fails( + tmp_path, monkeypatch, capsys +): + import openai + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + original = tmp_path / "original.md" + replacement = tmp_path / "replacement.md" + failing = tmp_path / "failing.md" + original.write_text("alpha original evidence", encoding="utf-8") + replacement.write_text("alpha replacement evidence", encoding="utf-8") + failing.write_text("beta second evidence", encoding="utf-8") + + filesystem = open_test_filesystem(workspace) + existing_ref = filesystem.register_file( + storage_uri=original.as_uri(), + external_id="doc-existing", + folder_path="/documents", + title=original.name, + content_type="text/markdown", + content=original.read_text(encoding="utf-8"), + metadata={"year": 2023}, + ) + baseline = registration_logical_state(workspace) + + class FailingBetaOpenAI(FakeOpenAI): + def create(self, *, model, input, dimensions): + if any("beta" in str(text).lower() for text in input): + raise RuntimeError("embedding unavailable for second registration") + return super().create(model=model, input=input, dimensions=dimensions) + + monkeypatch.setattr(openai, "OpenAI", FailingBetaOpenAI) + reopened = open_test_filesystem(workspace) + with pytest.raises(RuntimeError, match="summary projection.*second registration"): + reopened.register_files( + [ + { + "storage_uri": replacement.as_uri(), + "external_id": "doc-existing", + "folder_path": "/documents/replacement", + "title": replacement.name, + "content_type": "text/markdown", + "content": replacement.read_text(encoding="utf-8"), + "metadata": {"year": 2024}, + }, + { + "storage_uri": failing.as_uri(), + "external_id": "doc-failing", + "folder_path": "/documents/new", + "title": failing.name, + "content_type": "text/markdown", + "content": failing.read_text(encoding="utf-8"), + "metadata": {"year": 2025}, + }, + ] + ) + + assert registration_logical_state(workspace) == baseline + strict_reopen = open_test_filesystem(workspace) + assert strict_reopen.store.file_refs_for_scope() == [existing_ref] + entry = strict_reopen.store.get_file(existing_ref) + assert entry.metadata["year"] == 2023 + assert entry.storage_uri == original.as_uri() + assert strict_reopen.pageindex_structure(existing_ref)["available"] is True + for command in ( + ["stat", "/documents/original.md"], + ["cat", "/documents/original.md", "--structure"], + ["grep", "original", "/documents/original.md"], + ["browse", "/documents", "original"], + ): + assert main(["--workspace", str(workspace), *command]) == 0 + output = json.loads(capsys.readouterr().out) + assert output["success"] is True + + +def test_register_files_rollback_preserves_preexisting_catalog_projection_and_cache( + tmp_path, monkeypatch +): + import openai + + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + existing_dir = tmp_path / "existing" + batch_dir = tmp_path / "batch" + existing_dir.mkdir() + batch_dir.mkdir() + existing = existing_dir / "shared.md" + shared = batch_dir / "shared.md" + second = batch_dir / "second.md" + existing.write_text("alpha registration evidence", encoding="utf-8") + shared.write_text(existing.read_text(encoding="utf-8"), encoding="utf-8") + second.write_text("beta registration evidence", encoding="utf-8") + + filesystem = open_test_filesystem(workspace) + existing_ref = filesystem.register_file( + storage_uri=existing.as_uri(), + folder_path="/documents", + title=existing.name, + content_type="text/markdown", + content=existing.read_text(encoding="utf-8"), + metadata={"year": 2023}, + ) + filesystem.metadata.register_schema( + {"fields": {"year": {"description": "preexisting definition"}}}, + source="manual", + ) + baseline = registration_logical_state(workspace) + + class FailingBetaOpenAI(FakeOpenAI): + def create(self, *, model, input, dimensions): + if any("beta" in str(text).lower() for text in input): + raise RuntimeError("embedding unavailable for second registration") + return super().create(model=model, input=input, dimensions=dimensions) + + monkeypatch.setattr(openai, "OpenAI", FailingBetaOpenAI) + reopened = open_test_filesystem(workspace) + with pytest.raises(RuntimeError, match="summary projection.*second registration"): + reopened.register_files( + [ + { + "storage_uri": source.as_uri(), + "folder_path": "/documents/new", + "title": source.name, + "content_type": "text/markdown", + "content": source.read_text(encoding="utf-8"), + "metadata": {"year": 2024, "batch_only": "new"}, + } + for source in (shared, second) + ] + ) + + assert registration_logical_state(workspace) == baseline + strict_reopen = open_test_filesystem(workspace) + assert strict_reopen.store.get_file(existing_ref).metadata["year"] == 2023 + structure = strict_reopen.pageindex_structure(existing_ref) + assert structure["available"] is True + assert structure["structure"][0]["title"] == existing.stem + + +def test_register_files_rolls_back_partial_pageindex_preparation( + tmp_path, monkeypatch +): + from pageindex import PageIndexClient + + install_network_fakes(monkeypatch) + successful_index = PageIndexClient.index + calls = 0 + + def fail_after_second_pageindex_write(self, file_path, mode="auto"): + nonlocal calls + calls += 1 + document_id = successful_index(self, file_path, mode=mode) + if calls == 2: + raise RuntimeError("PageIndex extraction failed for second registration") + return document_id + + monkeypatch.setattr(PageIndexClient, "index", fail_after_second_pageindex_write) + workspace = tmp_path / "workspace" + filesystem = open_test_filesystem(workspace) + first = tmp_path / "first.md" + second = tmp_path / "second.md" + first.write_text("alpha registration evidence", encoding="utf-8") + second.write_text("beta registration evidence", encoding="utf-8") + baseline = registration_logical_state(workspace) + + with pytest.raises( + RuntimeError, + match="PageIndex extraction failed for second registration", + ): + filesystem.register_files( + [ + { + "storage_uri": source.as_uri(), + "folder_path": "/documents/new", + "title": source.name, + "content_type": "text/markdown", + "content": source.read_text(encoding="utf-8"), + "metadata": {"year": 2024}, + } + for source in (first, second) + ] + ) + + assert registration_logical_state(workspace) == baseline + reopened = open_test_filesystem(workspace) + assert reopened.store.folder_info("/")["path"] == "/" + assert reopened.store.file_refs_for_scope() == [] + + +def test_register_files_prepare_failure_restores_existing_owned_artifacts( + tmp_path, monkeypatch +): + from pageindex.filesystem.commands import PIFSCommandExecutor + + install_network_fakes(monkeypatch) + workspace = tmp_path / "workspace" + existing = tmp_path / "existing.md" + replacement = tmp_path / "replacement.md" + existing.write_text("alpha original registration evidence", encoding="utf-8") + replacement.write_text("beta replacement registration evidence", encoding="utf-8") + + filesystem = open_test_filesystem(workspace) + existing_ref = filesystem.register_file( + storage_uri=existing.as_uri(), + external_id="doc-existing", + folder_path="/documents", + title=existing.name, + content_type="text/markdown", + content=existing.read_text(encoding="utf-8"), + metadata={"year": 2023}, + ) + existing_entry = filesystem.store.get_file(existing_ref) + text_path = Path(existing_entry.text_artifact_path) + raw_path = Path(existing_entry.raw_artifact_path) + original_text = text_path.read_bytes() + original_raw = raw_path.read_bytes() + original_pageindex_doc_id = existing_entry.pageindex_doc_id + baseline = registration_logical_state(workspace) + + with pytest.raises(RuntimeError, match="requires PageIndex extraction"): + filesystem.register_files( + [ + { + "storage_uri": replacement.as_uri(), + "external_id": "doc-existing", + "folder_path": "/documents/replacement", + "title": replacement.name, + "content_type": "text/markdown", + "content": replacement.read_text(encoding="utf-8"), + "metadata": {"year": 2024}, + }, + { + "storage_uri": "https://example.invalid/unresolvable.md", + "external_id": "doc-failing", + "folder_path": "/documents/new", + "title": "unresolvable.md", + "content_type": "text/markdown", + "content": "must not be registered", + "metadata": {"year": 2025}, + }, + ] + ) + + assert registration_logical_state(workspace) == baseline + assert text_path.read_bytes() == original_text + assert raw_path.read_bytes() == original_raw + + reopened = open_test_filesystem(workspace) + reopened_entry = reopened.store.get_file(existing_ref) + assert reopened_entry.pageindex_doc_id == original_pageindex_doc_id + executor = PIFSCommandExecutor(reopened) + locator = "/documents/existing.md" + assert json.loads(executor.execute(f"stat {locator}"))["success"] is True + cat = json.loads(executor.execute(f"cat {locator} --structure")) + assert cat["success"] is True + assert cat["data"]["pagination"]["available"] is True + grep = json.loads(executor.execute(f"grep alpha {locator}")) + assert grep["success"] is True + assert grep["data"]["matches"] + structure = reopened.pageindex_structure(existing_ref) + assert structure["available"] is True + assert structure["structure"][0]["title"] == existing.stem + + +@pytest.mark.parametrize("raw_path_kind", ["canonical", "custom"]) +def test_register_files_rollback_follows_explicit_raw_artifact_management_policy( + raw_path_kind, tmp_path, monkeypatch +): + import openai + from pageindex.filesystem.store import make_file_ref + + install_network_fakes(monkeypatch) + + class FailingSecondOpenAI(FakeOpenAI): + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.calls = 0 + + def create(self, *, model, input, dimensions): + self.calls += 1 + if self.calls == 2: + raise RuntimeError("embedding unavailable for second registration") + return super().create(model=model, input=input, dimensions=dimensions) + + monkeypatch.setattr(openai, "OpenAI", FailingSecondOpenAI) + workspace = tmp_path / "workspace" + filesystem = open_test_filesystem(workspace) + first = tmp_path / "first.md" + second = tmp_path / "second.md" + first.write_text("alpha registration evidence", encoding="utf-8") + second.write_text("beta registration evidence", encoding="utf-8") + first_external_id = f"doc-{raw_path_kind}-raw" + first_file_ref = make_file_ref(first_external_id) + raw_dir = workspace / "artifacts" / "raw" + raw_dir.mkdir(parents=True, exist_ok=True) + raw_path = raw_dir / ( + f"{first_file_ref}.json" + if raw_path_kind == "canonical" + else "custom-sentinel.json" + ) + sentinel = f"{raw_path_kind} raw sentinel".encode() + raw_path.write_bytes(sentinel) + baseline = registration_logical_state(workspace) + + with pytest.raises(RuntimeError, match="summary projection.*second registration"): + filesystem.register_files( + [ + { + "storage_uri": first.as_uri(), + "external_id": first_external_id, + "folder_path": "/documents/new", + "title": first.name, + "content_type": "text/markdown", + "content": first.read_text(encoding="utf-8"), + "metadata": {"year": 2024}, + "raw_artifact_path": str(raw_path), + }, + { + "storage_uri": second.as_uri(), + "external_id": "doc-second-raw", + "folder_path": "/documents/new", + "title": second.name, + "content_type": "text/markdown", + "content": second.read_text(encoding="utf-8"), + "metadata": {"year": 2025}, + }, + ] + ) + + assert raw_path.read_bytes() == sentinel + assert registration_logical_state(workspace) == baseline + reopened = open_test_filesystem(workspace) + assert reopened.store.file_refs_for_scope() == [] + + +def test_register_file_rejects_non_json_metadata_before_side_effects( + tmp_path, monkeypatch +): + from pageindex import PageIndexClient + + install_network_fakes(monkeypatch) + successful_index = PageIndexClient.index + indexed_paths = [] + + def record_index(self, file_path, mode="auto"): + indexed_paths.append(Path(file_path).name) + return successful_index(self, file_path, mode=mode) + + monkeypatch.setattr(PageIndexClient, "index", record_index) + workspace = tmp_path / "workspace" + filesystem = open_test_filesystem(workspace) + source = tmp_path / "notes.md" + source.write_text("alpha registration evidence", encoding="utf-8") + baseline = registration_logical_state(workspace) + + with pytest.raises(ValueError, match="metadata must be JSON serializable"): + filesystem.register_file( + storage_uri=source.as_uri(), + folder_path="/documents/new", + title=source.name, + content_type="text/markdown", + content=source.read_text(encoding="utf-8"), + metadata={"not_json": object()}, + ) + + assert registration_logical_state(workspace) == baseline + assert indexed_paths == [] + reopened = open_test_filesystem(workspace) + assert reopened.store.folder_info("/")["path"] == "/" + assert reopened.store.file_refs_for_scope() == [] + + +def test_register_files_preflights_all_metadata_before_preparing_batch( + tmp_path, monkeypatch +): + from pageindex import PageIndexClient + + install_network_fakes(monkeypatch) + successful_index = PageIndexClient.index + indexed_paths = [] + + def record_index(self, file_path, mode="auto"): + indexed_paths.append(Path(file_path).name) + return successful_index(self, file_path, mode=mode) + + monkeypatch.setattr(PageIndexClient, "index", record_index) + workspace = tmp_path / "workspace" + filesystem = open_test_filesystem(workspace) + first = tmp_path / "first.md" + second = tmp_path / "second.md" + first.write_text("alpha registration evidence", encoding="utf-8") + second.write_text("beta registration evidence", encoding="utf-8") + baseline = registration_logical_state(workspace) + + with pytest.raises(ValueError, match="metadata must be JSON serializable"): + filesystem.register_files( + [ + { + "storage_uri": first.as_uri(), + "folder_path": "/documents/new", + "title": first.name, + "content_type": "text/markdown", + "content": first.read_text(encoding="utf-8"), + "metadata": {"year": 2024}, + }, + { + "storage_uri": second.as_uri(), + "folder_path": "/documents/new", + "title": second.name, + "content_type": "text/markdown", + "content": second.read_text(encoding="utf-8"), + "metadata": {"not_json": object()}, + }, + ] + ) + + assert registration_logical_state(workspace) == baseline + assert indexed_paths == [] + reopened = open_test_filesystem(workspace) + assert reopened.store.folder_info("/")["path"] == "/" + assert reopened.store.file_refs_for_scope() == [] + + +def test_register_file_rolls_back_partial_projection_creation_and_reopens( + tmp_path, monkeypatch +): + install_network_fakes(monkeypatch) + # SQLite's Unix VFS accepts summary.sqlite at this path length but rejects + # the eight-character-longer embedding_cache.sqlite path. + workspace = nested_path_with_length(tmp_path / "long-workspace", 453) + filesystem = open_test_filesystem(workspace) + source = tmp_path / "notes.md" + source.write_text("alpha registration evidence", encoding="utf-8") + + with pytest.raises(sqlite3.OperationalError, match="unable to open database file"): + filesystem.register_file( + storage_uri=source.as_uri(), + folder_path="/documents", + title=source.name, + content_type="text/markdown", + content=source.read_text(encoding="utf-8"), + ) + + projection_dir = workspace / "artifacts" / "projection_indexes" + assert not (projection_dir / "summary.sqlite").exists() + assert not (projection_dir / "embedding_cache.sqlite").exists() + assert catalog_file_count(workspace) == 0 + assert not list((workspace / "artifacts" / "text").glob("*")) + assert not list((workspace / "artifacts" / "raw").glob("*")) + + reopened = open_test_filesystem(workspace) + assert reopened.store.folder_info("/")["path"] == "/" + + +def test_cli_add_rejects_duplicate_target_without_changing_owned_document( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + first_dir = tmp_path / "first" + second_dir = tmp_path / "second" + first_dir.mkdir() + second_dir.mkdir() + first = first_dir / "notes.md" + second = second_dir / "notes.md" + first.write_text("first alpha evidence", encoding="utf-8") + second.write_text("second beta evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + command = ["--workspace", str(workspace), "add"] + + assert main([*command, str(first), "/documents"]) == 0 + capsys.readouterr() + assert main([*command, str(second), "/documents"]) == 1 + + assert "already exists" in capsys.readouterr().err + assert catalog_file_count(workspace) == 1 + owned = list((workspace / "artifacts" / "uploads").glob("*/notes.md")) + assert len(owned) == 1 + assert owned[0].read_text(encoding="utf-8") == "first alpha evidence" + + +def test_physical_browse_preserves_collision_aware_tree_locators( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + for filename, content in ( + ("alpha.md", "alpha evidence"), + ("beta.md", "beta evidence"), + ): + source = tmp_path / filename + source.write_text(content, encoding="utf-8") + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + with sqlite3.connect(workspace / "filesystem.sqlite") as connection: + connection.execute( + "UPDATE file_folders SET metadata_json = ?", + (json.dumps({"display_name": "same.md"}),), + ) + + assert main(["--workspace", str(workspace), "tree", "/documents", "-L", "1"]) == 0 + tree_files = json.loads(capsys.readouterr().out)["data"]["tree"]["files"] + tree_paths = {row["path"] for row in tree_files} + assert tree_paths == { + "/documents/same.md~1", + "/documents/same.md~2", + } + + assert main(["--workspace", str(workspace), "browse", "/documents", "alpha"]) == 0 + browse_documents = json.loads(capsys.readouterr().out)["data"]["documents"] + browse_paths = [row["path"] for row in browse_documents] + + assert len(browse_documents) == 2 + assert len(set(browse_paths)) == 2 + assert set(browse_paths) == tree_paths + for path in browse_paths: + assert main(["--workspace", str(workspace), "stat", path]) == 0 + stat = json.loads(capsys.readouterr().out)["data"]["document"] + assert stat["path"] == path + + +def test_browse_recursively_returns_one_global_ranked_page_of_ten( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + for index in range(12): + source = tmp_path / f"doc-{index:02d}.md" + relevance = "alpha" if index in {1, 10} else "beta" + source.write_text(f"{relevance} evidence {index}", encoding="utf-8") + folder = "/documents/a" if index % 2 == 0 else "/documents/b" + assert main(["--workspace", str(workspace), "add", str(source), folder]) == 0 + capsys.readouterr() + + command = [ + "--workspace", + str(workspace), + "browse", + "/documents", + "alpha", + "--recursive", + ] + assert main(command) == 0 + first = json.loads(capsys.readouterr().out)["data"] + assert len(first["documents"]) == 10 + assert first["pagination"] == { + "page": 1, + "page_size": 10, + "has_more": True, + "next_page": 2, + } + assert {row["title"] for row in first["documents"][:2]} == { + "doc-01.md", + "doc-10.md", + } + assert {row["folder_path"] for row in first["documents"]} == { + "/documents/a", + "/documents/b", + } + + assert main([*command, "--page", "2"]) == 0 + second = json.loads(capsys.readouterr().out)["data"] + assert len(second["documents"]) == 2 + assert second["pagination"] == { + "page": 2, + "page_size": 10, + "has_more": False, + "next_page": None, + } + assert not { + row["path"] for row in first["documents"] + }.intersection(row["path"] for row in second["documents"]) + + +def test_browse_respects_physical_and_virtual_scope( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + records = [ + ("current.md", "alpha current", "/documents/current", 2024), + ("old.md", "alpha old", "/documents/archive", 2023), + ] + for filename, content, folder, year in records: + source = tmp_path / filename + source.write_text(content, encoding="utf-8") + assert main(["--workspace", str(workspace), "add", str(source), folder]) == 0 + capsys.readouterr() + assert main( + [ + "--workspace", + str(workspace), + "setmeta", + f"{folder}/{filename}", + json.dumps({"year": year}), + ] + ) == 0 + capsys.readouterr() + + assert main( + [ + "--workspace", + str(workspace), + "browse", + "/documents/current", + "alpha", + ] + ) == 0 + physical = json.loads(capsys.readouterr().out)["data"]["documents"] + assert [row["title"] for row in physical] == ["current.md"] + + assert main( + [ + "--workspace", + str(workspace), + "browse", + "/documents/@year/2024", + "alpha", + ] + ) == 0 + virtual = json.loads(capsys.readouterr().out)["data"]["documents"] + assert [row["title"] for row in virtual] == ["current.md"] + assert virtual[0]["path"] == "/documents/@year/2024/current.md" + + +def test_browse_requires_query_and_an_existing_summary_projection( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "tree", "/", "-L", "1"]) == 0 + capsys.readouterr() + + assert main(["--workspace", str(workspace), "browse", "/documents"]) == 2 + missing_query = json.loads(capsys.readouterr().out) + assert "requires a query" in missing_query["error"]["message"] + + assert main(["--workspace", str(workspace), "browse", "/", "alpha"]) == 2 + missing_projection = json.loads(capsys.readouterr().out) + assert "Summary Projection is not available" in missing_projection["error"]["message"] + + +def test_browse_rejects_incompatible_embedding_identity_without_mutation( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + config = write_embedding_config(tmp_path, monkeypatch) + workspace = tmp_path / "workspace" + source = tmp_path / "notes.md" + source.write_text("alpha evidence", encoding="utf-8") + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + databases = [ + workspace / "filesystem.sqlite", + workspace / "artifacts" / "projection_indexes" / "summary.sqlite", + workspace / "artifacts" / "projection_indexes" / "embedding_cache.sqlite", + ] + before = {path: hashlib.sha256(path.read_bytes()).hexdigest() for path in databases} + config.write_text( + json.dumps( + { + "embedding_base_url": "https://example.invalid/v1", + "embedding_model": "different-model", + "embedding_dimensions": 3, + } + ), + encoding="utf-8", + ) + + assert main(["--workspace", str(workspace), "browse", "/documents", "alpha"]) == 2 + + payload = json.loads(capsys.readouterr().out) + assert "Incompatible PIFS Summary Embedding Profile" in payload["error"]["message"] + assert { + path: hashlib.sha256(path.read_bytes()).hexdigest() for path in databases + } == before + + +def test_agent_prompts_describe_only_the_retained_command_surface(): + from pageindex.filesystem.agent import ( + AGENT_SYSTEM_PROMPT, + AGENT_TOOL_POLICY, + BASH_TOOL_DESCRIPTION, + ) + + prompts = "\n".join( + [AGENT_SYSTEM_PROMPT, BASH_TOOL_DESCRIPTION, AGENT_TOOL_POLICY] + ) + for command in ("tree", "browse", "stat", "cat", "grep"): + assert command in prompts + for retired_contract in ( + "ls as an alias", + "file_ref", + "--space", + "Do not use find", + "recursive grep", + ): + assert retired_contract not in prompts + + +def test_cat_page_reads_are_bounded_and_grep_is_single_document( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + install_network_fakes(monkeypatch) + write_embedding_config(tmp_path, monkeypatch) + source = tmp_path / "notes.md" + source.write_text("line one\nalpha evidence", encoding="utf-8") + workspace = tmp_path / "workspace" + assert main(["--workspace", str(workspace), "add", str(source), "/documents"]) == 0 + capsys.readouterr() + + assert main( + [ + "--workspace", + str(workspace), + "cat", + "/documents/notes.md", + "--page", + "1", + ] + ) == 0 + page = json.loads(capsys.readouterr().out)["data"] + assert page["requested_pages"] == "1" + assert page["content"]["text"] == "line one\nalpha evidence" + + assert main( + [ + "--workspace", + str(workspace), + "cat", + "/documents/notes.md", + "--page", + "1-6", + ] + ) == 2 + too_wide = json.loads(capsys.readouterr().out) + assert "at most 5 pages" in too_wide["error"]["message"] + + assert main( + ["--workspace", str(workspace), "grep", "alpha", "/documents"] + ) == 2 + folder_grep = json.loads(capsys.readouterr().out) + assert "resolved file locator" in folder_grep["error"]["message"] + + +@pytest.mark.parametrize( + "command", + [ + ["ls", "/"], + ["find", "/"], + ["browse", "/", "alpha", "--space", "entity"], + ["browse", "/", "alpha", "--limit", "2"], + ["stat", "/", "--schema"], + ["stat", "/", "--field", "year"], + ["stat", "/one", "/two"], + ["cat", "/document"], + ["cat", "/document", "--all"], + ["cat", "/document", "--range", "1-2"], + ["grep", "alpha", "/document", "--recursive"], + ], +) +def test_retired_command_forms_return_structured_errors(command, tmp_path, capsys): + from pageindex.filesystem.cli import main + + status = main(["--workspace", str(tmp_path / "workspace"), *command]) + + payload = json.loads(capsys.readouterr().out) + assert status == 2 + assert payload["success"] is False + assert payload["error"]["code"] == "invalid_command" + + +def test_set_workspace_preserves_embedding_api_key_without_exposing_or_persisting_it_elsewhere( + tmp_path, monkeypatch, capsys +): + from pageindex.filesystem.cli import main + + config = tmp_path / "pifs.json" + config.write_text( + json.dumps( + { + "embedding_base_url": "https://example.invalid/v1", + "embedding_model": "test-embedding", + "embedding_dimensions": 3, + "embedding_timeout": 12, + "embedding_api_key": "config-secret", + "embedding_provider": "legacy-provider", + } + ), + encoding="utf-8", + ) + monkeypatch.setenv("PIFS_CONFIG_FILE", str(config)) + workspace = tmp_path / "workspace" + + assert main(["set", "workspace", str(workspace)]) == 0 + output = capsys.readouterr().out + + assert json.loads(config.read_text(encoding="utf-8")) == { + "embedding_api_key": "config-secret", + "embedding_base_url": "https://example.invalid/v1", + "embedding_dimensions": "3", + "embedding_model": "test-embedding", + "embedding_timeout": "12", + "workspace": str(workspace), + } + assert "config-secret" not in output + + +def test_public_example_is_a_short_supported_cli_walkthrough(): + example = Path(__file__).parents[1] / "examples" / "pifs_demo.py" + source = example.read_text(encoding="utf-8") + + assert len(source.splitlines()) <= 60 + for command in ("add", "tree", "browse", "stat", "cat", "grep"): + assert f'"{command}"' in source + for retired_surface in ( + "embedding_provider", + "metadata_provider", + "file_ref", + "SummaryProjectionIndexer", + "shutil.rmtree", + '"ls"', + ): + assert retired_surface not in source diff --git a/tests/test_semantic_index.py b/tests/test_semantic_index.py new file mode 100644 index 000000000..0346f60f9 --- /dev/null +++ b/tests/test_semantic_index.py @@ -0,0 +1,220 @@ +import hashlib + +import pytest + +from pageindex.filesystem.semantic_index import ( + SemanticIndexRecord, + SQLiteVecSemanticIndex, +) +from pageindex.filesystem.semantic_projection import ( + EmbeddingCache, + SummaryEmbeddingProfile, + SummaryProjection, +) + + +class FixedEmbedder: + def __init__(self, dimensions: int): + self.dimensions = dimensions + + def embed(self, texts): + return [[1.0, *([0.0] * (self.dimensions - 1))] for _ in texts] + + +def test_sqlite_vec_semantic_index_round_trip(tmp_path): + index = SQLiteVecSemanticIndex(tmp_path / "semantic.sqlite") + index.reset(dimension=3, metadata={"kind": "summary"}) + index.upsert_many( + [ + SemanticIndexRecord( + file_ref="file_a", + external_id="doc_a", + source_type="documents", + title="Multipart upload limits", + text="multipart upload limits", + vector=[1.0, 0.0, 0.0], + metadata={"topic": "uploads"}, + ), + SemanticIndexRecord( + file_ref="file_b", + external_id="doc_b", + source_type="documents", + title="GPU cache issue", + text="gpu cache issue", + vector=[0.0, 1.0, 0.0], + metadata={"topic": "runtime"}, + ), + ] + ) + + assert [ + item.external_id for item in index.search([0.9, 0.1, 0.0], limit=2) + ] == ["doc_a", "doc_b"] + + +def test_sqlite_vec_file_ref_filter_is_not_limited_by_global_rank(tmp_path): + index = SQLiteVecSemanticIndex(tmp_path / "semantic.sqlite") + index.reset(dimension=2, metadata={"kind": "summary"}) + records = [ + SemanticIndexRecord( + file_ref=f"file_off_{item:02d}", + external_id=f"doc_off_{item:02d}", + source_type="documents", + title=f"Off scope {item:02d}", + text="off scope", + vector=[1.0, 0.0], + ) + for item in range(30) + ] + records.append( + SemanticIndexRecord( + file_ref="file_in_scope", + external_id="doc_in_scope", + source_type="documents", + title="In scope", + text="in scope", + vector=[0.0, 1.0], + ) + ) + index.upsert_many(records) + + results = index.search( + [1.0, 0.0], + limit=1, + filters={"file_ref": ["file_in_scope"]}, + ) + + assert [item.file_ref for item in results] == ["file_in_scope"] + + +def test_summary_projection_owns_write_search_and_delete_lifecycle(tmp_path): + profile = SummaryEmbeddingProfile( + base_url="https://EXAMPLE.invalid/v1/", + model="fake", + dimensions=3, + api_key="runtime-only", + ) + projection = SummaryProjection( + tmp_path / "projection", + profile=profile, + embedder=FixedEmbedder(3), + create=True, + ) + record = { + "file_ref": "file_a", + "external_id": "doc_a", + "source_type": "documents", + "title": "A", + "metadata": {"summary": "Unified summary", "department": "ops"}, + } + + assert projection.upsert_summary(record)["status"] == "ready" + assert projection.info()["embedding_identity"] == { + "base_url": "https://example.invalid/v1", + "model": "fake", + "dimensions": 3, + } + assert [candidate.file_ref for candidate in projection.search("summary")] == [ + "file_a" + ] + assert projection.delete_summary("file_a") == 1 + assert projection.available is False + + +def test_projection_profile_mismatch_preserves_existing_database(tmp_path): + index_dir = tmp_path / "projection" + profile = SummaryEmbeddingProfile( + base_url="https://example.invalid/v1", + model="fake", + dimensions=3, + api_key="runtime-only", + ) + projection = SummaryProjection( + index_dir, + profile=profile, + embedder=FixedEmbedder(3), + create=True, + ) + projection.upsert_summary( + { + "file_ref": "file_a", + "external_id": "doc_a", + "source_type": "documents", + "title": "A", + "metadata": {"summary": "Preserve me"}, + } + ) + summary_path = index_dir / "summary.sqlite" + before = hashlib.sha256(summary_path.read_bytes()).hexdigest() + mismatch = SummaryEmbeddingProfile( + base_url="https://example.invalid/v1", + model="different", + dimensions=3, + api_key="runtime-only", + ) + + with pytest.raises(Exception, match="Incompatible PIFS Summary Embedding Profile"): + SummaryProjection(index_dir, profile=mismatch, create=False) + + assert hashlib.sha256(summary_path.read_bytes()).hexdigest() == before + + +def test_embedding_cache_rejects_response_length_mismatch(tmp_path): + class ShortEmbedder: + def embed(self, texts): + return [[1.0, 0.0, 0.0]] + + cache = EmbeddingCache(tmp_path / "cache.sqlite", create=True) + profile = SummaryEmbeddingProfile( + base_url="https://example.invalid/v1", + model="fake", + dimensions=3, + api_key="runtime-only", + ) + + with pytest.raises(ValueError, match="embedding response length mismatch"): + cache.embed_texts( + ["first", "second"], + profile=profile, + embedder=ShortEmbedder(), + batch_size=2, + ) + + +def test_embed_with_retry_only_retries_transient_errors(monkeypatch): + from pageindex.filesystem.semantic_projection import embed_with_retry + + sleeps = [] + + class EmbeddingError(Exception): + def __init__(self, status_code): + self.status_code = status_code + + class FlakyEmbedder: + calls = 0 + + def embed(self, texts): + self.calls += 1 + if self.calls == 1: + raise EmbeddingError(500) + return [[1.0, 0.0, 0.0]] + + monkeypatch.setattr( + "pageindex.filesystem.semantic_projection.time.sleep", sleeps.append + ) + embedder = FlakyEmbedder() + assert embed_with_retry(embedder, ["text"]) == [[1.0, 0.0, 0.0]] + assert embedder.calls == 2 + assert sleeps == [1.0] + + class PermanentEmbedder: + calls = 0 + + def embed(self, texts): + self.calls += 1 + raise EmbeddingError(401) + + permanent = PermanentEmbedder() + with pytest.raises(EmbeddingError): + embed_with_retry(permanent, ["text"]) + assert permanent.calls == 1