feat(filesystem): add PageIndex FileSystem and PIFS CLI

.github/workflows/codeql.yml

@@ -14,7 +14,7 @@ permissions:
   actions: read
 
 jobs:
-  analyze:
+  analyze-actions:
     name: Analyze (actions)
     runs-on: ubuntu-latest
     timeout-minutes: 30
@@ -32,3 +32,22 @@ jobs:
         uses: github/codeql-action/analyze@v4
         with:
           category: /language:actions
+
+  analyze-python:
+    name: Analyze (python)
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v4
+        with:
+          languages: python
+          build-mode: none
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v4
+        with:
+          category: /language:python

.gitignore

@@ -3,4 +3,12 @@ __pycache__
 .DS_Store
 .env*
 .venv/
+.claude/
+.codex/
 logs/
+examples/pifs_workspace/
+examples/
+/CONTEXT.md
+/docs/pifs-core-alignment/
+/pyproject.toml
+/uv.lock

README.md

@@ -175,7 +175,7 @@ python3 run_pageindex.py --pdf_path /path/to/your/document.pdf
 You can customize the processing with additional optional arguments:
 
 ```
---model                 LLM model to use (default: gpt-4o-2024-11-20)
+--model                 LLM model to use (default: gpt-5.4)
 --toc-check-pages       Pages to check for table of contents (default: 20)
 --max-pages-per-node    Max pages per node (default: 10)
 --max-tokens-per-node   Max tokens per node (default: 20000)

examples/pifs_demo.py

@@ -0,0 +1,46 @@
+"""A minimal PageIndex FileSystem CLI walkthrough.
+
+Configure the Summary Embedding Profile in your PIFS config and provide its API
+key through ``PIFS_EMBEDDING_API_KEY`` before running this example.
+"""
+
+from __future__ import annotations
+
+import argparse
+import subprocess
+from pathlib import Path
+
+
+def run(workspace: Path, document: Path, query: str) -> int:
+    target = f"/documents/{document.name}"
+    commands = [
+        ["add", str(document), "/documents"],
+        ["tree", "/documents", "-L", "1"],
+        ["browse", "/documents", query],
+        ["stat", target],
+        ["cat", target, "--structure"],
+        ["cat", target, "--page", "1"],
+        ["grep", query, target],
+    ]
+    for command in commands:
+        print(f"\n$ pifs {' '.join(command)}")
+        result = subprocess.run(
+            ["pifs", "--workspace", str(workspace), *command],
+            check=False,
+        )
+        if result.returncode:
+            return result.returncode
+    return 0
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("document", type=Path, help="PDF or Markdown document")
+    parser.add_argument("--workspace", type=Path, default=Path("pifs-workspace"))
+    parser.add_argument("--query", default="key findings")
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    raise SystemExit(run(args.workspace, args.document, args.query))

pageindex/__init__.py

@@ -1,4 +1,22 @@
-from .page_index import *
-from .page_index_md import md_to_tree
-from .retrieve import get_document, get_document_structure, get_page_content
-from .client import PageIndexClient
+import os
+
+os.environ.setdefault("LITELLM_LOCAL_MODEL_COST_MAP", "true")
+
+_OPTIONAL_CORE_IMPORTS = {"litellm", "openai", "PyPDF2", "pymupdf"}
+
+try:
+    from .page_index import *
+    from .page_index_md import md_to_tree
+    from .retrieve import get_document, get_document_structure, get_page_content
+    from .client import PageIndexClient
+except ModuleNotFoundError as exc:
+    if exc.name not in _OPTIONAL_CORE_IMPORTS:
+        raise
+
+
+def __getattr__(name: str):
+    if name == "PageIndexFileSystem":
+        from .filesystem import PageIndexFileSystem
+
+        return PageIndexFileSystem
+    raise AttributeError(f"module {__name__!r} has no attribute {name!r}")

pageindex/config.yaml

@@ -1,4 +1,4 @@
-model: "gpt-4o-2024-11-20"
+model: "gpt-5.4"
 # model: "anthropic/claude-sonnet-4-6"
 retrieve_model: "gpt-5.4"  # defaults to `model` if not set
 toc_check_page_num: 20
@@ -7,4 +7,4 @@ max_token_num_each_node: 20000
 if_add_node_id: "yes"
 if_add_node_summary: "yes"
 if_add_doc_description: "no"
-if_add_node_text: "no"
+if_add_node_text: "no"

pageindex/filesystem/__init__.py

@@ -0,0 +1,3 @@
+from .core import PageIndexFileSystem
+
+__all__ = ["PageIndexFileSystem"]

pageindex/filesystem/_embedding_identity.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from urllib.parse import urlsplit, urlunsplit
+
+
+DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1"
+
+
+def normalize_base_url(value: str | None) -> str:
+    raw = str(value or DEFAULT_OPENAI_BASE_URL).strip()
+    parsed = urlsplit(raw)
+    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+        raise ValueError("embedding base URL must be an absolute HTTP(S) URL")
+    if parsed.username or parsed.password or parsed.query or parsed.fragment:
+        raise ValueError(
+            "embedding base URL must not contain credentials, query parameters, or a fragment"
+        )
+    return urlunsplit(
+        (
+            parsed.scheme.lower(),
+            parsed.netloc.lower(),
+            parsed.path.rstrip("/"),
+            "",
+            "",
+        )
+    )
+
+
+def normalize_model(value: str | None) -> str:
+    model = str(value or "").strip()
+    if not model:
+        raise ValueError("embedding model must not be empty")
+    return model

pageindex/filesystem/_projection_topology.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+
+def projection_database_paths(index_dir: str | Path) -> tuple[Path, Path]:
+    index_dir = Path(index_dir).expanduser()
+    return index_dir / "summary.sqlite", index_dir / "embedding_cache.sqlite"
+
+
+def projection_database_path_present(path: str | Path) -> bool:
+    path = Path(path)
+    return path.exists() or path.is_symlink()
+
+
+def projection_database_pair(
+    index_dir: str | Path,
+) -> tuple[Path, Path] | None:
+    summary_path, cache_path = projection_database_paths(index_dir)
+    summary_exists = projection_database_path_present(summary_path)
+    cache_exists = projection_database_path_present(cache_path)
+    if not summary_exists and not cache_exists:
+        return None
+    if not summary_exists or not cache_exists:
+        raise RuntimeError(
+            "PIFS Summary Projection topology is incomplete; migrate this workspace with "
+            "pifs-data/scripts/migrate_pifs_workspace.py before opening it."
+        )
+    return summary_path, cache_path

pageindex/filesystem/_sqlite_schema.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+import sqlite3
+from typing import Any
+
+
+def regular_table_names(connection: sqlite3.Connection) -> set[str]:
+    return {
+        str(row[0])
+        for row in connection.execute(
+            "SELECT name FROM sqlite_master "
+            "WHERE type = 'table' AND name NOT LIKE 'sqlite_%'"
+        )
+    }
+
+
+def sqlite_schema_signature(
+    connection: sqlite3.Connection,
+    tables: set[str],
+) -> dict[str, Any]:
+    columns = {
+        table: tuple(
+            tuple(row)
+            for row in connection.execute(f'PRAGMA table_xinfo("{table}")')
+        )
+        for table in sorted(tables)
+    }
+    indexes: dict[str, tuple[tuple[Any, ...], ...]] = {}
+    foreign_keys: dict[str, tuple[tuple[Any, ...], ...]] = {}
+    for table in sorted(tables):
+        table_indexes = []
+        for row in connection.execute(f'PRAGMA index_list("{table}")'):
+            name = str(row[1])
+            origin = str(row[3])
+            index_columns = tuple(
+                str(column[2])
+                for column in connection.execute(f'PRAGMA index_info("{name}")')
+            )
+            table_indexes.append(
+                (
+                    name if origin == "c" else None,
+                    int(row[2]),
+                    origin,
+                    int(row[4]),
+                    index_columns,
+                )
+            )
+        indexes[table] = tuple(sorted(table_indexes, key=repr))
+        foreign_keys[table] = tuple(
+            sorted(
+                (
+                    str(row[2]),
+                    str(row[3]),
+                    str(row[4]),
+                    str(row[5]),
+                    str(row[6]),
+                    str(row[7]),
+                )
+                for row in connection.execute(f'PRAGMA foreign_key_list("{table}")')
+            )
+        )
+    return {
+        "tables": tables,
+        "columns": columns,
+        "indexes": indexes,
+        "foreign_keys": foreign_keys,
+    }
+
+
+def normalized_table_sql(connection: sqlite3.Connection, table: str) -> str:
+    row = connection.execute(
+        "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = ?",
+        (table,),
+    ).fetchone()
+    return " ".join(str(row[0] if row else "").lower().split())

pageindex/filesystem/_workspace_consistency.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+import sqlite3
+from pathlib import Path
+
+
+def _readonly_connection(path: Path) -> sqlite3.Connection:
+    connection = sqlite3.connect(f"{path.resolve().as_uri()}?mode=ro", uri=True)
+    connection.row_factory = sqlite3.Row
+    return connection
+
+
+def _inconsistent_workspace_error(detail: str) -> RuntimeError:
+    return RuntimeError(
+        f"Inconsistent PIFS workspace: {detail}; migrate this workspace with "
+        "pifs-data/scripts/migrate_pifs_workspace.py before opening it."
+    )
+
+
+def validate_catalog_root(catalog_path: str | Path) -> None:
+    try:
+        with _readonly_connection(Path(catalog_path)) as catalog:
+            root = catalog.execute(
+                "SELECT folder_id, parent_id, name FROM folders WHERE path = '/'"
+            ).fetchone()
+    except sqlite3.Error as exc:
+        raise _inconsistent_workspace_error(
+            "the catalog canonical root folder could not be read"
+        ) from exc
+    if (
+        root is None
+        or root["folder_id"] != "folder_root"
+        or root["parent_id"] is not None
+        or root["name"] != "/"
+    ):
+        raise _inconsistent_workspace_error(
+            "the catalog is missing its canonical root folder"
+        )
+
+
+def validate_workspace_consistency(
+    catalog_path: str | Path,
+    summary_path: str | Path,
+) -> None:
+    catalog_path = Path(catalog_path)
+    validate_catalog_root(catalog_path)
+    try:
+        with _readonly_connection(catalog_path) as catalog:
+            active_file_refs = {
+                str(row[0])
+                for row in catalog.execute(
+                    "SELECT file_ref FROM files WHERE deleted_at IS NULL"
+                )
+            }
+        with _readonly_connection(Path(summary_path)) as summary:
+            projected_file_refs = {
+                str(row[0])
+                for row in summary.execute(
+                    "SELECT file_ref FROM semantic_index_docs"
+                )
+            }
+    except sqlite3.Error as exc:
+        raise _inconsistent_workspace_error(
+            "catalog and Summary Projection references could not be read"
+        ) from exc
+    orphaned = sorted(projected_file_refs - active_file_refs)
+    if orphaned:
+        shown = ", ".join(orphaned[:5])
+        raise _inconsistent_workspace_error(
+            "Summary Projection file_ref values do not reference active catalog files: "
+            f"{shown}"
+        )
+    missing = sorted(active_file_refs - projected_file_refs)
+    if missing:
+        shown = ", ".join(missing[:5])
+        raise _inconsistent_workspace_error(
+            "active catalog file_ref values are missing from the Summary Projection: "
+            f"{shown}"
+        )
+
+
+def validate_catalog_without_projection(catalog_path: str | Path) -> None:
+    catalog_path = Path(catalog_path)
+    validate_catalog_root(catalog_path)
+    try:
+        with _readonly_connection(catalog_path) as catalog:
+            active_file_refs = sorted(
+                str(row[0])
+                for row in catalog.execute(
+                    "SELECT file_ref FROM files WHERE deleted_at IS NULL"
+                )
+            )
+    except sqlite3.Error as exc:
+        raise _inconsistent_workspace_error(
+            "active catalog references could not be read"
+        ) from exc
+    if active_file_refs:
+        shown = ", ".join(active_file_refs[:5])
+        raise _inconsistent_workspace_error(
+            "active catalog files require a complete Summary Projection: "
+            f"{shown}"
+        )